From 0aa2029b0323c1b7fe5b364a23315ccb22d4fc5b Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 21 Jun 2023 19:06:43 -0400 Subject: [PATCH 01/40] integrate openconfig progess --- bittensor/__init__.py | 4 +- bittensor/_cli/__init__.py | 9 - bittensor/_config/__init__.py | 194 ------------------ bittensor/_config/config_impl.py | 189 ----------------- bittensor/_subtensor/__init__.py | 8 +- bittensor/_wallet/__init__.py | 4 +- requirements/prod.txt | 1 + tests/integration_tests/test_cli.py | 15 +- .../integration_tests/test_cli_no_network.py | 7 - 9 files changed, 12 insertions(+), 419 deletions(-) delete mode 100644 bittensor/_config/__init__.py delete mode 100644 bittensor/_config/config_impl.py diff --git a/bittensor/__init__.py b/bittensor/__init__.py index 0157ebb47c..5fcfa788f0 100644 --- a/bittensor/__init__.py +++ b/bittensor/__init__.py @@ -131,7 +131,7 @@ def turn_console_off(): # ---- Config ---- -from bittensor._config import config as config +from openconfig import config as config # ---- LOGGING ---- # Duplicate import for ease of use. @@ -169,7 +169,7 @@ def turn_console_off(): # ---- Classes ----- from bittensor._cli.cli_impl import CLI as CLI -from bittensor._config.config_impl import Config as Config +from openconfig.config_impl import Config as Config from bittensor._subtensor.chain_data import DelegateInfo as DelegateInfo from bittensor._wallet.wallet_impl import Wallet as Wallet from bittensor._keyfile.keyfile_impl import Keyfile as Keyfile diff --git a/bittensor/_cli/__init__.py b/bittensor/_cli/__init__.py index 8d727a9696..c3465db79c 100644 --- a/bittensor/_cli/__init__.py +++ b/bittensor/_cli/__init__.py @@ -87,15 +87,6 @@ def __create_parser__() -> 'argparse.ArgumentParser': return parser - @staticmethod - def config(args: List[str]) -> 'bittensor.config': - """ From the argument parser, add config to bittensor.executor and local config - Return: bittensor.config object - """ - parser = cli.__create_parser__() - - return parser - @staticmethod def config(args: List[str]) -> 'bittensor.config': """ From the argument parser, add config to bittensor.executor and local config diff --git a/bittensor/_config/__init__.py b/bittensor/_config/__init__.py deleted file mode 100644 index e26a611a06..0000000000 --- a/bittensor/_config/__init__.py +++ /dev/null @@ -1,194 +0,0 @@ -""" -Create and init the config class, which manages the config of different bittensor modules. -""" -# The MIT License (MIT) -# Copyright © 2021 Yuma Rao -# Copyright © 2022 Opentensor Foundation - -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the “Software”), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. - -# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -import os -import sys -from argparse import ArgumentParser, Namespace -from typing import List, Optional, Dict - -import bittensor -import yaml -from loguru import logger -import pandas as pd - -from . import config_impl - -logger = logger.opt(colors=True) - -class config: - """ - Create and init the config class, which manages the config of different bittensor modules. - """ - class InvalidConfigFile(Exception): - """ In place of YAMLError - """ - - def __new__( cls, parser: ArgumentParser = None, strict: bool = False, args: Optional[List[str]] = None ): - r""" Translates the passed parser into a nested Bittensor config. - Args: - parser (argparse.Parser): - Command line parser object. - strict (bool): - If true, the command line arguments are strictly parsed. - args (list of str): - Command line arguments. - Returns: - config (bittensor.Config): - Nested config object created from parser arguments. - """ - if parser == None: - return config_impl.Config() - - # Optionally add config specific arguments - try: - parser.add_argument('--config', type=str, help='If set, defaults are overridden by passed file.') - except: - # this can fail if the --config has already been added. - pass - try: - parser.add_argument('--strict', action='store_true', help='''If flagged, config will check that only exact arguemnts have been set.''', default=False ) - except: - # this can fail if the --config has already been added. - pass - - # Get args from argv if not passed in. - if args == None: - args = sys.argv[1:] - - # 1.1 Optionally load defaults if the --config is set. - try: - config_file_path = str(os.getcwd()) + '/' + vars(parser.parse_known_args(args)[0])['config'] - except Exception as e: - config_file_path = None - - # Parse args not strict - params = cls.__parse_args__(args=args, parser=parser, strict=False) - - # 2. Optionally check for --strict, if stict we will parse the args strictly. - strict = params.strict - - if config_file_path != None: - config_file_path = os.path.expanduser(config_file_path) - try: - with open(config_file_path) as f: - params_config = yaml.safe_load(f) - print('Loading config defaults from: {}'.format(config_file_path)) - parser.set_defaults(**params_config) - except Exception as e: - print('Error in loading: {} using default parser settings'.format(e)) - - # 2. Continue with loading in params. - params = cls.__parse_args__(args=args, parser=parser, strict=strict) - - _config = config_impl.Config() - - # Splits params on dot syntax i.e neuron.axon_port - for arg_key, arg_val in params.__dict__.items(): - split_keys = arg_key.split('.') - head = _config - keys = split_keys - while len(keys) > 1: - if hasattr(head, keys[0]): - head = getattr(head, keys[0]) - keys = keys[1:] - else: - head[keys[0]] = config_impl.Config() - head = head[keys[0]] - keys = keys[1:] - if len(keys) == 1: - head[keys[0]] = arg_val - - # Get defaults for this config - is_set_map = cls.__fill_is_set_list__(_config, bittensor.defaults) - - _config['__is_set'] = is_set_map - - _config.__fill_with_defaults__(is_set_map, bittensor.defaults) - - return _config - - @staticmethod - def __fill_is_set_list__(_config: 'bittensor.Config', defaults: 'bittensor.Config') -> Dict[str, bool]: - """Creates an is_set map - Args: - _config (bittensor.Config): - Config to generate is_set mapping. - defaults (bittensor.Config): - The bittensor defaults - Returns: - is_set_map (Dict[str, bool]): - A map from flattened param name to whether this param was set in a flag. - """ - is_set_map = {} - config_d = _config.__dict__ - # Only defaults we are concerned with - defaults_filtered = {} - for key in config_d.keys(): - if key in defaults.keys(): - defaults_filtered[key] = getattr(defaults, key) - # Avoid erroring out if defaults aren't set for a submodule - if defaults_filtered == {}: - return is_set_map - - flat_config = pd.json_normalize(config_d, sep='.').to_dict('records')[0] - flat_defaults = pd.json_normalize(defaults_filtered, sep='.').to_dict('records')[0] - for key, _ in flat_defaults.items(): - if key in flat_config: - is_set_map[key] = True - else: - is_set_map[key] = False - - return is_set_map - - - @staticmethod - def __parse_args__( args: List[str], parser: ArgumentParser = None, strict: bool = False) -> Namespace: - """Parses the passed args use the passed parser. - Args: - args (List[str]): - List of arguments to parse. - parser (argparse.ArgumentParser): - Command line parser object. - strict (bool): - If true, the command line arguments are strictly parsed. - Returns: - Namespace: - Namespace object created from parser arguments. - """ - if not strict: - params = parser.parse_known_args(args=args)[0] - else: - params = parser.parse_args(args=args) - - return params - - @staticmethod - def full(): - """ From the parser, add arguments to multiple bittensor sub-modules - """ - parser = ArgumentParser() - bittensor.wallet.add_args( parser ) - bittensor.subtensor.add_args( parser ) - bittensor.axon.add_args( parser ) - bittensor.metagraph.add_args( parser ) - bittensor.dataset.add_args( parser ) - bittensor.prometheus.add_args( parser ) - return bittensor.config( parser ) diff --git a/bittensor/_config/config_impl.py b/bittensor/_config/config_impl.py deleted file mode 100644 index ee07678fe9..0000000000 --- a/bittensor/_config/config_impl.py +++ /dev/null @@ -1,189 +0,0 @@ -""" -Implementation of the config class, which manages the config of different bittensor modules. -""" -# The MIT License (MIT) -# Copyright © 2021 Yuma Rao -# Copyright © 2022 Opentensor Foundation - -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the “Software”), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. - -# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -import yaml -import json -import pandas -import bittensor -from munch import Munch -from prometheus_client import Info -from pandas import json_normalize -from typing import Dict -import copy -import bittensor - -class Config ( Munch ): - """ - Implementation of the config class, which manages the config of different bittensor modules. - """ - __is_set: Dict[str, bool] - - def __init__(self, loaded_config = None ): - super().__init__() - if loaded_config: - raise NotImplementedError('Function load_from_relative_path is not fully implemented.') - - def __repr__(self) -> str: - return self.__str__() - - def __str__(self) -> str: - config_dict = self.toDict() - config_dict.pop("__is_set") - return "\n" + yaml.dump(config_dict) - - def to_string(self, items) -> str: - """ Get string from items - """ - return "\n" + yaml.dump(items.toDict()) - - def update_with_kwargs( self, kwargs ): - """ Add config to self - """ - for key,val in kwargs.items(): - self[key] = val - - @classmethod - def _merge( cls, a, b ): - """Merge two configurations recursively. - If there is a conflict, the value from the second configuration will take precedence. - """ - for key in b: - if key in a: - if isinstance( a[key], dict ) and isinstance( b[key], dict ): - a[key] = cls._merge( a[key], b[key] ) - else: - a[key] = b[key] - else: - a[key] = b[key] - return a - - def merge(self, b): - """ Merge two configs - """ - self = self._merge( self, b ) - - def to_prometheus(self): - """ - Sends the config to the inprocess prometheus server if it exists. - """ - try: - prometheus_info = Info('config', 'Config Values') - # Make copy, remove __is_set map - config_copy = copy.deepcopy(self) - - del config_copy['__is_set'] - - config_info = json_normalize(json.loads(json.dumps(config_copy)), sep='.').to_dict(orient='records')[0] - formatted_info = {} - for key in config_info: - config_info[key] = str(config_info[key]) - formatted_info[key.replace('.', '_')] = str(config_info[key]) - prometheus_info.info(formatted_info) - except ValueError: - # The user called this function twice in the same session. - # TODO(const): need a way of distinguishing the various config items. - bittensor.__console__.print("The config has already been added to prometheus.", highlight=True) - - def is_set(self, param_name: str) -> bool: - """ - Returns a boolean indicating whether the parameter has been set or is still the default. - """ - if param_name not in self.get('__is_set'): - return False - else: - return self.get('__is_set')[param_name] - - def __fill_with_defaults__(self, is_set_map: Dict[str, bool], defaults: 'Config') -> None: - """ - Recursively fills the config with the default values using is_set_map - """ - defaults_filtered = {} - for key in self.keys(): - if key in defaults.keys(): - defaults_filtered[key] = getattr(defaults, key) - # Avoid erroring out if defaults aren't set for a submodule - if defaults_filtered == {}: return - - flat_defaults = json_normalize(defaults_filtered, sep='.').to_dict('records')[0] - for key, val in flat_defaults.items(): - if key not in is_set_map: - continue - elif not is_set_map[key]: - # If the key is not set, set it to the default value - # Loop through flattened key to get leaf - a = self - keys = key.split('.') - for key_ in keys[:-1]: - if key_ not in a: - a[key_] = {} - a = a[key_] - # Set leaf to default value - a[keys[-1]] = val - - def to_defaults(self): - try: - if 'axon' in self.keys(): - bittensor.defaults.axon.port = self.axon.port - bittensor.defaults.axon.ip = self.axon.ip - bittensor.defaults.axon.external_port = self.axon.external_port - bittensor.defaults.axon.external_ip = self.axon.external_ip - bittensor.defaults.axon.max_workers = self.axon.max_workers - bittensor.defaults.axon.maximum_concurrent_rpcs = self.axon.maximum_concurrent_rpcs - - if 'dataset' in self.keys(): - bittensor.defaults.dataset.batch_size = self.dataset.batch_size - bittensor.defaults.dataset.block_size = self.dataset.block_size - bittensor.defaults.dataset.num_batches = self.dataset.num_batches - bittensor.defaults.dataset.num_workers = self.dataset.num_workers - bittensor.defaults.dataset.dataset_names = self.dataset.dataset_names - bittensor.defaults.dataset.data_dir = self.dataset.data_dir - bittensor.defaults.dataset.save_dataset = self.dataset.save_dataset - bittensor.defaults.dataset.max_datasets = self.dataset.max_datasets - - if 'logging' in self.keys(): - bittensor.defaults.logging.debug = self.logging.debug - bittensor.defaults.logging.trace = self.logging.trace - bittensor.defaults.logging.record_log = self.logging.record_log - bittensor.defaults.logging.logging_dir = self.logging.logging_dir - - if 'subtensor' in self.keys(): - bittensor.defaults.subtensor.network = self.subtensor.network - bittensor.defaults.subtensor.chain_endpoint = self.subtensor.chain_endpoint - - if 'threadpool' in self.keys(): - bittensor.defaults.threadpool.max_workers = self.threadpool.max_workers - bittensor.defaults.threadpool.maxsize = self.threadpool.maxsize - - if 'wallet' in self.keys(): - bittensor.defaults.wallet.name = self.wallet.name - bittensor.defaults.wallet.hotkey = self.wallet.hotkey - bittensor.defaults.wallet.path = self.wallet.path - - if 'wandb' in self.keys(): - bittensor.defaults.wandb.name = self.wandb.name - bittensor.defaults.wandb.project = self.wandb.project - bittensor.defaults.wandb.tags = self.wandb.tags - bittensor.defaults.wandb.run_group = self.wandb.run_group - bittensor.defaults.wandb.directory = self.wandb.directory - bittensor.defaults.wandb.offline = self.wandb.offline - - except Exception as e: - print('Error when loading config into defaults {}'.format(e)) \ No newline at end of file diff --git a/bittensor/_subtensor/__init__.py b/bittensor/_subtensor/__init__.py index ef8b943ee1..fa207a5970 100644 --- a/bittensor/_subtensor/__init__.py +++ b/bittensor/_subtensor/__init__.py @@ -146,7 +146,7 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ): setattr(bittensor.defaults, prefix, bittensor.Config()) getattr(bittensor.defaults, prefix).subtensor = bittensor.defaults.subtensor try: - parser.add_argument('--' + prefix_str + 'subtensor.network', default = argparse.SUPPRESS, type=str, + parser.add_argument('--' + prefix_str + 'subtensor.network', default = bittensor.defaults.subtensor.network, type=str, help='''The subtensor network flag. The likely choices are: -- finney (main network) -- local (local running network) @@ -165,10 +165,10 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ): parser.add_argument('--' + prefix_str + 'subtensor.register.verbose', help="Whether to ouput the registration statistics verbosely.", action='store_true', required=False, default=bittensor.defaults.subtensor.register.verbose) ## Registration args for CUDA registration. - parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.use_cuda', '--' + prefix_str + 'cuda', '--' + prefix_str + 'cuda.use_cuda', default=argparse.SUPPRESS, help='''Set flag to use CUDA to register.''', action="store_true", required=False ) - parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.no_cuda', '--' + prefix_str + 'no_cuda', '--' + prefix_str + 'cuda.no_cuda', dest=prefix_str + 'subtensor.register.cuda.use_cuda', default=argparse.SUPPRESS, help='''Set flag to not use CUDA for registration''', action="store_false", required=False ) + parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.use_cuda', '--' + prefix_str + 'cuda', '--' + prefix_str + 'cuda.use_cuda', default=bittensor.defaults.subtensor.register.cuda.use_cuda, help='''Set flag to use CUDA to register.''', action="store_true", required=False ) + parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.no_cuda', '--' + prefix_str + 'no_cuda', '--' + prefix_str + 'cuda.no_cuda', dest=prefix_str + 'subtensor.register.cuda.use_cuda', default=not bittensor.defaults.subtensor.register.cuda.use_cuda, help='''Set flag to not use CUDA for registration''', action="store_false", required=False ) - parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.dev_id', '--' + prefix_str + 'cuda.dev_id', type=int, nargs='+', default=argparse.SUPPRESS, help='''Set the CUDA device id(s). Goes by the order of speed. (i.e. 0 is the fastest).''', required=False ) + parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.dev_id', '--' + prefix_str + 'cuda.dev_id', type=int, nargs='+', default=bittensor.defaults.subtensor.register.cuda.dev_id, help='''Set the CUDA device id(s). Goes by the order of speed. (i.e. 0 is the fastest).''', required=False ) parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.TPB', '--' + prefix_str + 'cuda.TPB', type=int, default=bittensor.defaults.subtensor.register.cuda.TPB, help='''Set the number of Threads Per Block for CUDA.''', required=False ) except argparse.ArgumentError: diff --git a/bittensor/_wallet/__init__.py b/bittensor/_wallet/__init__.py index 3bb036de3a..bee000a4b5 100644 --- a/bittensor/_wallet/__init__.py +++ b/bittensor/_wallet/__init__.py @@ -112,8 +112,8 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ): setattr(bittensor.defaults, prefix, bittensor.Config()) getattr(bittensor.defaults, prefix).wallet = bittensor.defaults.wallet try: - parser.add_argument('--' + prefix_str + 'wallet.name', required=False, default=argparse.SUPPRESS, help='''The name of the wallet to unlock for running bittensor (name mock is reserved for mocking this wallet)''') - parser.add_argument('--' + prefix_str + 'wallet.hotkey', required=False, default=argparse.SUPPRESS, help='''The name of wallet's hotkey.''') + parser.add_argument('--' + prefix_str + 'wallet.name', required=False, default=bittensor.defaults.wallet.name, help='''The name of the wallet to unlock for running bittensor (name mock is reserved for mocking this wallet)''') + parser.add_argument('--' + prefix_str + 'wallet.hotkey', required=False, default=bittensor.defaults.wallet.hotkey, help='''The name of wallet's hotkey.''') parser.add_argument('--' + prefix_str + 'wallet.path', required=False, default=bittensor.defaults.wallet.path, help='''The path to your bittensor wallets''') parser.add_argument('--' + prefix_str + 'wallet._mock', action='store_true', default=bittensor.defaults.wallet._mock, help='To turn on wallet mocking for testing purposes.') diff --git a/requirements/prod.txt b/requirements/prod.txt index 63df59bce5..9068095753 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -19,6 +19,7 @@ munch==2.5.0 nest_asyncio==1.5.6 netaddr==0.8.0 numpy>=1.21.6,<=1.24.2 +openconfig==0.0.0 pandas==1.5.2 password_strength==0.0.3.post2 prometheus_client==0.14.1 diff --git a/tests/integration_tests/test_cli.py b/tests/integration_tests/test_cli.py index 7ac3ed3de8..e2bf0dda39 100644 --- a/tests/integration_tests/test_cli.py +++ b/tests/integration_tests/test_cli.py @@ -20,11 +20,8 @@ import unittest from copy import deepcopy from types import SimpleNamespace -from typing import Dict, Optional, Tuple +from typing import Dict from unittest.mock import MagicMock, patch -import time - -import os import random @@ -38,7 +35,8 @@ def generate_wallet(coldkey: "Keypair" = None, hotkey: "Keypair" = None): - wallet = bittensor.wallet(_mock=True).create() + _config = TestCLIWithNetworkAndConfig.construct_config() + wallet = bittensor.wallet(_config, _mock=True).create() if not coldkey: coldkey = Keypair.create_from_mnemonic(Keypair.generate_mnemonic()) @@ -97,13 +95,6 @@ def config(self): def construct_config(): defaults = bittensor.Config() - # Get defaults for this config - is_set_map = bittensor.config.__fill_is_set_list__(defaults, bittensor.defaults) - - defaults['__is_set'] = is_set_map - - defaults.__fill_with_defaults__(is_set_map, bittensor.defaults) - defaults.netuid = 1 bittensor.subtensor.add_defaults(defaults) # Always use mock subtensor. diff --git a/tests/integration_tests/test_cli_no_network.py b/tests/integration_tests/test_cli_no_network.py index ee2dc81f0e..d6b687ddd2 100644 --- a/tests/integration_tests/test_cli_no_network.py +++ b/tests/integration_tests/test_cli_no_network.py @@ -70,13 +70,6 @@ def config(self): def construct_config(): defaults = bittensor.Config() - # Get defaults for this config - is_set_map = bittensor.config.__fill_is_set_list__(defaults, bittensor.defaults) - - defaults['__is_set'] = is_set_map - - defaults.__fill_with_defaults__(is_set_map, bittensor.defaults) - defaults.netuid = 1 bittensor.subtensor.add_defaults( defaults ) defaults.subtensor.network = 'mock' From e038d2a27f9a8177bd2d3a73cc425bfe54f45676 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 21 Jun 2023 20:28:07 -0400 Subject: [PATCH 02/40] add type or suppress for argparse --- bittensor/utils/__init__.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 05d7c2f6bc..db2cedc439 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -1,11 +1,12 @@ import numbers -from typing import Callable, Union, List, Optional, Dict +from typing import Callable, Union, List, Optional, Dict, Literal, Type, Any import bittensor import pandas import requests import torch import scalecodec +import argparse from substrateinterface import Keypair from substrateinterface.utils import ss58 from .registration import create_pow @@ -148,7 +149,7 @@ def get_ss58_format( ss58_address: str ) -> int: """Returns the ss58 format of the given ss58 address.""" return ss58.get_ss58_format( ss58_address ) -def strtobool_with_default( default: bool ) -> Callable[[str], bool]: +def strtobool_with_default( default: bool ) -> Callable[[str], Union[bool, Literal['==SUPRESS==']]]: """ Creates a strtobool function with a default value. @@ -161,7 +162,7 @@ def strtobool_with_default( default: bool ) -> Callable[[str], bool]: return lambda x: strtobool(x) if x != "" else default -def strtobool(val: str) -> bool: +def strtobool(val: str) -> Union[bool, Literal['==SUPRESS==']]: """ Converts a string to a boolean value. @@ -241,3 +242,9 @@ def u8_key_to_ss58(u8_key: List[int]) -> str: """ # First byte is length, then 32 bytes of key. return scalecodec.ss58_encode( bytes(u8_key).hex(), bittensor.__ss58_format__) + +def type_or_suppress(type: Type) -> Callable[[Any], Union[Any, Literal['==SUPRESS==']]]: + def _type_or_suppress(value: Any) -> Union[Any, Literal['==SUPRESS==']]: + return value if value == argparse.SUPPRESS else type(value) + + return _type_or_suppress From 0f8700f9de8345e5670b8c1cd988c1a7e64caa38 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 21 Jun 2023 20:35:55 -0400 Subject: [PATCH 03/40] use for problematic types --- bittensor/_cli/commands/overview.py | 3 ++- bittensor/_subtensor/__init__.py | 9 +++++---- bittensor/_wallet/__init__.py | 5 ++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/bittensor/_cli/commands/overview.py b/bittensor/_cli/commands/overview.py index 8f86539f6c..23cd61e89f 100644 --- a/bittensor/_cli/commands/overview.py +++ b/bittensor/_cli/commands/overview.py @@ -23,6 +23,7 @@ from rich.table import Table from rich.prompt import Prompt from typing import List, Optional, Dict +from bittensor.utils import type_or_suppress from .utils import get_hotkey_wallets_for_wallet, get_coldkey_wallets_for_path, get_all_wallets_for_path console = bittensor.__console__ @@ -275,7 +276,7 @@ def add_args( parser: argparse.ArgumentParser ): '--width', dest='width', action='store', - type=int, + type=type_or_suppress(int), help='''Set the output width of the overview. Defaults to automatic width from terminal.''', default=None, ) diff --git a/bittensor/_subtensor/__init__.py b/bittensor/_subtensor/__init__.py index fa207a5970..ae27ca88b5 100644 --- a/bittensor/_subtensor/__init__.py +++ b/bittensor/_subtensor/__init__.py @@ -25,6 +25,7 @@ from loguru import logger from substrateinterface import SubstrateInterface from torch.cuda import is_available as is_cuda_available +from bittensor.utils import type_or_suppress from . import subtensor_impl, subtensor_mock logger = logger.opt(colors=True) @@ -159,8 +160,8 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ): ''') parser.add_argument('--' + prefix_str + 'subtensor._mock', action='store_true', help='To turn on subtensor mocking for testing purposes.', default=bittensor.defaults.subtensor._mock) # registration args. Used for register and re-register and anything that calls register. - parser.add_argument('--' + prefix_str + 'subtensor.register.num_processes', '-n', dest=prefix_str + 'subtensor.register.num_processes', help="Number of processors to use for registration", type=int, default=bittensor.defaults.subtensor.register.num_processes) - parser.add_argument('--' + prefix_str + 'subtensor.register.update_interval', '--' + prefix_str + 'subtensor.register.cuda.update_interval', '--' + prefix_str + 'cuda.update_interval', '-u', help="The number of nonces to process before checking for next block during registration", type=int, default=bittensor.defaults.subtensor.register.update_interval) + parser.add_argument('--' + prefix_str + 'subtensor.register.num_processes', '-n', dest=prefix_str + 'subtensor.register.num_processes', help="Number of processors to use for registration", type=type_or_suppress(int), default=bittensor.defaults.subtensor.register.num_processes) + parser.add_argument('--' + prefix_str + 'subtensor.register.update_interval', '--' + prefix_str + 'subtensor.register.cuda.update_interval', '--' + prefix_str + 'cuda.update_interval', '-u', help="The number of nonces to process before checking for next block during registration", type=type_or_suppress(int), default=bittensor.defaults.subtensor.register.update_interval) parser.add_argument('--' + prefix_str + 'subtensor.register.no_output_in_place', '--' + prefix_str + 'no_output_in_place', dest="subtensor.register.output_in_place", help="Whether to not ouput the registration statistics in-place. Set flag to disable output in-place.", action='store_false', required=False, default=bittensor.defaults.subtensor.register.output_in_place) parser.add_argument('--' + prefix_str + 'subtensor.register.verbose', help="Whether to ouput the registration statistics verbosely.", action='store_true', required=False, default=bittensor.defaults.subtensor.register.verbose) @@ -168,8 +169,8 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ): parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.use_cuda', '--' + prefix_str + 'cuda', '--' + prefix_str + 'cuda.use_cuda', default=bittensor.defaults.subtensor.register.cuda.use_cuda, help='''Set flag to use CUDA to register.''', action="store_true", required=False ) parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.no_cuda', '--' + prefix_str + 'no_cuda', '--' + prefix_str + 'cuda.no_cuda', dest=prefix_str + 'subtensor.register.cuda.use_cuda', default=not bittensor.defaults.subtensor.register.cuda.use_cuda, help='''Set flag to not use CUDA for registration''', action="store_false", required=False ) - parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.dev_id', '--' + prefix_str + 'cuda.dev_id', type=int, nargs='+', default=bittensor.defaults.subtensor.register.cuda.dev_id, help='''Set the CUDA device id(s). Goes by the order of speed. (i.e. 0 is the fastest).''', required=False ) - parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.TPB', '--' + prefix_str + 'cuda.TPB', type=int, default=bittensor.defaults.subtensor.register.cuda.TPB, help='''Set the number of Threads Per Block for CUDA.''', required=False ) + parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.dev_id', '--' + prefix_str + 'cuda.dev_id', type=type_or_suppress(int), nargs='+', default=bittensor.defaults.subtensor.register.cuda.dev_id, help='''Set the CUDA device id(s). Goes by the order of speed. (i.e. 0 is the fastest).''', required=False ) + parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.TPB', '--' + prefix_str + 'cuda.TPB', type=type_or_suppress(int), default=bittensor.defaults.subtensor.register.cuda.TPB, help='''Set the number of Threads Per Block for CUDA.''', required=False ) except argparse.ArgumentError: # re-parsing arguments. diff --git a/bittensor/_wallet/__init__.py b/bittensor/_wallet/__init__.py index bee000a4b5..79422c885e 100644 --- a/bittensor/_wallet/__init__.py +++ b/bittensor/_wallet/__init__.py @@ -17,11 +17,10 @@ import argparse import copy -from distutils.util import strtobool import os import bittensor -from bittensor.utils import strtobool +from bittensor.utils import strtobool, type_or_suppress from . import wallet_impl, wallet_mock class wallet: @@ -117,7 +116,7 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ): parser.add_argument('--' + prefix_str + 'wallet.path', required=False, default=bittensor.defaults.wallet.path, help='''The path to your bittensor wallets''') parser.add_argument('--' + prefix_str + 'wallet._mock', action='store_true', default=bittensor.defaults.wallet._mock, help='To turn on wallet mocking for testing purposes.') - parser.add_argument('--' + prefix_str + 'wallet.reregister', required=False, action='store', default=bittensor.defaults.wallet.reregister, type=strtobool, help='''Whether to reregister the wallet if it is not already registered.''') + parser.add_argument('--' + prefix_str + 'wallet.reregister', required=False, action='store', default=bittensor.defaults.wallet.reregister, type=type_or_suppress(strtobool), help='''Whether to reregister the wallet if it is not already registered.''') except argparse.ArgumentError as e: pass From b9cf48b1a7dea98562161a276eace99befa9b461 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 21 Jun 2023 20:36:00 -0400 Subject: [PATCH 04/40] fix type hints --- bittensor/utils/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index db2cedc439..ed8d7db31f 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -243,8 +243,8 @@ def u8_key_to_ss58(u8_key: List[int]) -> str: # First byte is length, then 32 bytes of key. return scalecodec.ss58_encode( bytes(u8_key).hex(), bittensor.__ss58_format__) -def type_or_suppress(type: Type) -> Callable[[Any], Union[Any, Literal['==SUPRESS==']]]: - def _type_or_suppress(value: Any) -> Union[Any, Literal['==SUPRESS==']]: - return value if value == argparse.SUPPRESS else type(value) +def type_or_suppress(type_func: Callable[[str], Any]) -> Callable[[str], Union[Any, Literal['==SUPRESS==']]]: + def _type_or_suppress(value: str) -> Union[Any, Literal['==SUPRESS==']]: + return value if value == argparse.SUPPRESS else type_func(value) return _type_or_suppress From f29f177363c3c7cf4d2c82836ef3b93ec373049b Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 21 Jun 2023 20:55:03 -0400 Subject: [PATCH 05/40] move test under testcase class --- .../test_priority_thread_pool.py | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/tests/integration_tests/test_priority_thread_pool.py b/tests/integration_tests/test_priority_thread_pool.py index d6ff31f391..b8d62cf343 100644 --- a/tests/integration_tests/test_priority_thread_pool.py +++ b/tests/integration_tests/test_priority_thread_pool.py @@ -16,17 +16,21 @@ # DEALINGS IN THE SOFTWARE. import bittensor - -priority_pool = bittensor.prioritythreadpool(max_workers=1) - -def test_priority_thread_pool(): - save = [] - def save_number(number,save): - save += [number] - with priority_pool: - for x in range(10): - priority_pool.submit(save_number, x,save,priority=x) - - assert save[0] == 0 - assert save[1] == 9 +import unittest + +class TestPriorityThreadPool(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.priority_pool = bittensor.prioritythreadpool(max_workers=1) + + def test_priority_thread_pool(self): + save = [] + def save_number(number,save): + save += [number] + with self.priority_pool: + for x in range(10): + self.priority_pool.submit(save_number, x,save,priority=x) + + assert save[0] == 0 + assert save[1] == 9 From 6d75d3b50f45b3c1892bf5d3b2addd3471bd68d1 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 21 Jun 2023 20:58:20 -0400 Subject: [PATCH 06/40] move tests under testcase class --- tests/unit_tests/bittensor_tests/test_axon.py | 186 +++++++++--------- 1 file changed, 96 insertions(+), 90 deletions(-) diff --git a/tests/unit_tests/bittensor_tests/test_axon.py b/tests/unit_tests/bittensor_tests/test_axon.py index 6b3d74cccd..74c5bf06b3 100644 --- a/tests/unit_tests/bittensor_tests/test_axon.py +++ b/tests/unit_tests/bittensor_tests/test_axon.py @@ -26,49 +26,10 @@ import bittensor from bittensor.utils.test_utils import get_random_unused_port -wallet = bittensor.wallet.mock() -axon = bittensor.axon( wallet = wallet, metagraph = None ) - -sender_wallet = bittensor.wallet.mock() def gen_nonce(): return f"{time.monotonic_ns()}" -def test_axon_start(): - mock_wallet = MagicMock( - spec=bittensor.Wallet, - coldkey=MagicMock(), - coldkeypub=MagicMock( - # mock ss58 address - ss58_address="5DD26kC2kxajmwfbbZmVmxhrY9VeeyR1Gpzy9i8wxLUg6zxm" - ), - hotkey=MagicMock( - ss58_address="5CtstubuSoVLJGCXkiWRNKrrGg2DVBZ9qMs2qYTLsZR4q1Wg" - ), - ) - axon = bittensor.axon( wallet = mock_wallet, metagraph = None ) - axon.start() - assert axon.server._state.stage == grpc._server._ServerStage.STARTED - -def test_axon_stop(): - mock_wallet = MagicMock( - spec=bittensor.Wallet, - coldkey=MagicMock(), - coldkeypub=MagicMock( - # mock ss58 address - ss58_address="5DD26kC2kxajmwfbbZmVmxhrY9VeeyR1Gpzy9i8wxLUg6zxm" - ), - hotkey=MagicMock( - ss58_address="5CtstubuSoVLJGCXkiWRNKrrGg2DVBZ9qMs2qYTLsZR4q1Wg" - ), - ) - axon = bittensor.axon( wallet = mock_wallet, metagraph = None ) - axon.start() - time.sleep( 1 ) - axon.stop() - time.sleep( 1 ) - assert axon.server._state.stage == grpc._server._ServerStage.STOPPED - def sign_v2(sender_wallet, receiver_wallet): nonce, receptor_uid = gen_nonce(), str(uuid.uuid1()) sender_hotkey = sender_wallet.hotkey.ss58_address @@ -80,9 +41,6 @@ def sign_v2(sender_wallet, receiver_wallet): def sign(sender_wallet, receiver_wallet, receiver_version): return sign_v2(sender_wallet, receiver_wallet) -def test_sign_v2(): - sign_v2(sender_wallet, wallet) - def is_port_in_use(port): import socket with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: @@ -92,54 +50,102 @@ def is_port_in_use(port): else: return False -def test_axon_is_destroyed(): - mock_wallet = MagicMock( - spec=bittensor.Wallet, - coldkey=MagicMock(), - coldkeypub=MagicMock( - # mock ss58 address - ss58_address="5DD26kC2kxajmwfbbZmVmxhrY9VeeyR1Gpzy9i8wxLUg6zxm" - ), - hotkey=MagicMock( - ss58_address="5CtstubuSoVLJGCXkiWRNKrrGg2DVBZ9qMs2qYTLsZR4q1Wg" - ), - ) - - port = get_random_unused_port() - assert is_port_in_use( port ) == False - axon = bittensor.axon ( wallet = mock_wallet, metagraph = None, port = port ) - assert is_port_in_use( port ) == True - axon.start() - assert is_port_in_use( port ) == True - axon.stop() - assert is_port_in_use( port ) == False - axon.__del__() - assert is_port_in_use( port ) == False - - port = get_random_unused_port() - assert is_port_in_use( port ) == False - axon2 = bittensor.axon ( wallet = mock_wallet, metagraph = None, port = port ) - assert is_port_in_use( port ) == True - axon2.start() - assert is_port_in_use( port ) == True - axon2.__del__() - assert is_port_in_use( port ) == False - - port_3 = get_random_unused_port() - assert is_port_in_use( port_3 ) == False - axonA = bittensor.axon ( wallet = mock_wallet, metagraph = None, port = port_3 ) - assert is_port_in_use( port_3 ) == True - axonB = bittensor.axon ( wallet = mock_wallet, metagraph = None, port = port_3 ) - assert axonA.server != axonB.server - assert is_port_in_use( port_3 ) == True - axonA.start() - assert is_port_in_use( port_3 ) == True - axonB.start() - assert is_port_in_use( port_3 ) == True - axonA.__del__() - assert is_port_in_use( port ) == False - axonB.__del__() - assert is_port_in_use( port ) == False +class TestAxon(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.wallet = wallet = bittensor.wallet.mock() + + cls.axon = bittensor.axon( wallet = wallet, metagraph = None ) + + cls.sender_wallet = bittensor.wallet.mock() + + + def test_axon_start(self): + mock_wallet = MagicMock( + spec=bittensor.Wallet, + coldkey=MagicMock(), + coldkeypub=MagicMock( + # mock ss58 address + ss58_address="5DD26kC2kxajmwfbbZmVmxhrY9VeeyR1Gpzy9i8wxLUg6zxm" + ), + hotkey=MagicMock( + ss58_address="5CtstubuSoVLJGCXkiWRNKrrGg2DVBZ9qMs2qYTLsZR4q1Wg" + ), + ) + axon = bittensor.axon( wallet = mock_wallet, metagraph = None ) + axon.start() + assert axon.server._state.stage == grpc._server._ServerStage.STARTED + + def test_axon_stop(self): + mock_wallet = MagicMock( + spec=bittensor.Wallet, + coldkey=MagicMock(), + coldkeypub=MagicMock( + # mock ss58 address + ss58_address="5DD26kC2kxajmwfbbZmVmxhrY9VeeyR1Gpzy9i8wxLUg6zxm" + ), + hotkey=MagicMock( + ss58_address="5CtstubuSoVLJGCXkiWRNKrrGg2DVBZ9qMs2qYTLsZR4q1Wg" + ), + ) + axon = bittensor.axon( wallet = mock_wallet, metagraph = None ) + axon.start() + time.sleep( 1 ) + axon.stop() + time.sleep( 1 ) + assert axon.server._state.stage == grpc._server._ServerStage.STOPPED + + def test_sign_v2(self): + sign_v2(self.sender_wallet, self.wallet) + + def test_axon_is_destroyed(self): + mock_wallet = MagicMock( + spec=bittensor.Wallet, + coldkey=MagicMock(), + coldkeypub=MagicMock( + # mock ss58 address + ss58_address="5DD26kC2kxajmwfbbZmVmxhrY9VeeyR1Gpzy9i8wxLUg6zxm" + ), + hotkey=MagicMock( + ss58_address="5CtstubuSoVLJGCXkiWRNKrrGg2DVBZ9qMs2qYTLsZR4q1Wg" + ), + ) + + port = get_random_unused_port() + assert is_port_in_use( port ) == False + axon = bittensor.axon ( wallet = mock_wallet, metagraph = None, port = port ) + assert is_port_in_use( port ) == True + axon.start() + assert is_port_in_use( port ) == True + axon.stop() + assert is_port_in_use( port ) == False + axon.__del__() + assert is_port_in_use( port ) == False + + port = get_random_unused_port() + assert is_port_in_use( port ) == False + axon2 = bittensor.axon ( wallet = mock_wallet, metagraph = None, port = port ) + assert is_port_in_use( port ) == True + axon2.start() + assert is_port_in_use( port ) == True + axon2.__del__() + assert is_port_in_use( port ) == False + + port_3 = get_random_unused_port() + assert is_port_in_use( port_3 ) == False + axonA = bittensor.axon ( wallet = mock_wallet, metagraph = None, port = port_3 ) + assert is_port_in_use( port_3 ) == True + axonB = bittensor.axon ( wallet = mock_wallet, metagraph = None, port = port_3 ) + assert axonA.server != axonB.server + assert is_port_in_use( port_3 ) == True + axonA.start() + assert is_port_in_use( port_3 ) == True + axonB.start() + assert is_port_in_use( port_3 ) == True + axonA.__del__() + assert is_port_in_use( port ) == False + axonB.__del__() + assert is_port_in_use( port ) == False # test external axon args class TestExternalAxon(unittest.TestCase): From a1f0f643768463d94c88bb10f146751d3e08bd50 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 21 Jun 2023 22:22:56 -0400 Subject: [PATCH 07/40] add test for btcli with empty arguments on every command --- .../integration_tests/test_cli_no_network.py | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/tests/integration_tests/test_cli_no_network.py b/tests/integration_tests/test_cli_no_network.py index d6b687ddd2..2014d2576e 100644 --- a/tests/integration_tests/test_cli_no_network.py +++ b/tests/integration_tests/test_cli_no_network.py @@ -341,6 +341,86 @@ class ExitEarlyException(Exception): assert cli.config.subtensor.register.cuda.use_cuda == False +class MockException(Exception): + pass + + +class TestEmptyArgs(unittest.TestCase): + """ + Test that the CLI doesn't crash when no args are passed + """ + _patched_subtensor = None + + @classmethod + def setUpClass(cls) -> None: + mock_delegate_info = { + "hotkey_ss58": "", + "total_stake": bittensor.Balance.from_rao(0), + "nominators": [], + "owner_ss58": "", + "take": 0.18, + "validator_permits": [], + "registrations": [], + "return_per_1000": bittensor.Balance.from_rao(0), + "total_daily_return": bittensor.Balance.from_rao(0) + } + cls._patched_subtensor = patch('bittensor._subtensor.subtensor_mock.MockSubtensor.__new__', new=MagicMock( + return_value=MagicMock( + get_subnets=MagicMock(return_value=[1]), # Mock subnet 1 ONLY. + block=10_000, + get_delegates=MagicMock(return_value=[ + bittensor.DelegateInfo( **mock_delegate_info ) + ]), + ) + )) + cls._patched_subtensor.start() + + @classmethod + def tearDownClass(cls) -> None: + cls._patched_subtensor.stop() + + def setUp(self): + self._config = TestCLINoNetwork.construct_config() + + @property + def config(self): + copy_ = deepcopy(self._config) + return copy_ + + @staticmethod + def construct_config(): + defaults = bittensor.Config() + + defaults.netuid = 1 + bittensor.subtensor.add_defaults( defaults ) + defaults.subtensor.network = 'mock' + defaults.no_version_checking = True + bittensor.axon.add_defaults( defaults ) + bittensor.wallet.add_defaults( defaults ) + bittensor.dataset.add_defaults( defaults ) + + return defaults + + @patch('rich.prompt.PromptBase.ask', side_effect=MockException) + def test_command_no_args(self, patched_prompt_ask): + # Get argparser + parser = bittensor.cli.__create_parser__() + # Get all commands from argparser + commands = [ + command for command in parser._actions[1].choices + ] + + # Test that each command can be run with no args + for command in commands: + try: + bittensor.cli(args=[ + command + ]).run() + except MockException: + pass # Expected exception + + # Should not raise any other exceptions + class TestCLIDefaultsNoNetwork(unittest.TestCase): _patched_subtensor = None From 8a221dea4c24fca27f47df8b0c3a8e067098f3f3 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 21 Jun 2023 23:27:26 -0400 Subject: [PATCH 08/40] remove unneeded setup from test --- .../integration_tests/test_cli_no_network.py | 40 ------------------- 1 file changed, 40 deletions(-) diff --git a/tests/integration_tests/test_cli_no_network.py b/tests/integration_tests/test_cli_no_network.py index 2014d2576e..95efbbded5 100644 --- a/tests/integration_tests/test_cli_no_network.py +++ b/tests/integration_tests/test_cli_no_network.py @@ -353,53 +353,13 @@ class TestEmptyArgs(unittest.TestCase): @classmethod def setUpClass(cls) -> None: - mock_delegate_info = { - "hotkey_ss58": "", - "total_stake": bittensor.Balance.from_rao(0), - "nominators": [], - "owner_ss58": "", - "take": 0.18, - "validator_permits": [], - "registrations": [], - "return_per_1000": bittensor.Balance.from_rao(0), - "total_daily_return": bittensor.Balance.from_rao(0) - } cls._patched_subtensor = patch('bittensor._subtensor.subtensor_mock.MockSubtensor.__new__', new=MagicMock( - return_value=MagicMock( - get_subnets=MagicMock(return_value=[1]), # Mock subnet 1 ONLY. - block=10_000, - get_delegates=MagicMock(return_value=[ - bittensor.DelegateInfo( **mock_delegate_info ) - ]), - ) )) cls._patched_subtensor.start() @classmethod def tearDownClass(cls) -> None: cls._patched_subtensor.stop() - - def setUp(self): - self._config = TestCLINoNetwork.construct_config() - - @property - def config(self): - copy_ = deepcopy(self._config) - return copy_ - - @staticmethod - def construct_config(): - defaults = bittensor.Config() - - defaults.netuid = 1 - bittensor.subtensor.add_defaults( defaults ) - defaults.subtensor.network = 'mock' - defaults.no_version_checking = True - bittensor.axon.add_defaults( defaults ) - bittensor.wallet.add_defaults( defaults ) - bittensor.dataset.add_defaults( defaults ) - - return defaults @patch('rich.prompt.PromptBase.ask', side_effect=MockException) def test_command_no_args(self, patched_prompt_ask): From af0bc343a92d76059ecaafa05e8b143ffe8d5c95 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 21 Jun 2023 23:30:07 -0400 Subject: [PATCH 09/40] remove type or suppress from int --- bittensor/_cli/commands/overview.py | 3 +-- bittensor/_subtensor/__init__.py | 9 ++++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/bittensor/_cli/commands/overview.py b/bittensor/_cli/commands/overview.py index 23cd61e89f..8f86539f6c 100644 --- a/bittensor/_cli/commands/overview.py +++ b/bittensor/_cli/commands/overview.py @@ -23,7 +23,6 @@ from rich.table import Table from rich.prompt import Prompt from typing import List, Optional, Dict -from bittensor.utils import type_or_suppress from .utils import get_hotkey_wallets_for_wallet, get_coldkey_wallets_for_path, get_all_wallets_for_path console = bittensor.__console__ @@ -276,7 +275,7 @@ def add_args( parser: argparse.ArgumentParser ): '--width', dest='width', action='store', - type=type_or_suppress(int), + type=int, help='''Set the output width of the overview. Defaults to automatic width from terminal.''', default=None, ) diff --git a/bittensor/_subtensor/__init__.py b/bittensor/_subtensor/__init__.py index ae27ca88b5..fa207a5970 100644 --- a/bittensor/_subtensor/__init__.py +++ b/bittensor/_subtensor/__init__.py @@ -25,7 +25,6 @@ from loguru import logger from substrateinterface import SubstrateInterface from torch.cuda import is_available as is_cuda_available -from bittensor.utils import type_or_suppress from . import subtensor_impl, subtensor_mock logger = logger.opt(colors=True) @@ -160,8 +159,8 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ): ''') parser.add_argument('--' + prefix_str + 'subtensor._mock', action='store_true', help='To turn on subtensor mocking for testing purposes.', default=bittensor.defaults.subtensor._mock) # registration args. Used for register and re-register and anything that calls register. - parser.add_argument('--' + prefix_str + 'subtensor.register.num_processes', '-n', dest=prefix_str + 'subtensor.register.num_processes', help="Number of processors to use for registration", type=type_or_suppress(int), default=bittensor.defaults.subtensor.register.num_processes) - parser.add_argument('--' + prefix_str + 'subtensor.register.update_interval', '--' + prefix_str + 'subtensor.register.cuda.update_interval', '--' + prefix_str + 'cuda.update_interval', '-u', help="The number of nonces to process before checking for next block during registration", type=type_or_suppress(int), default=bittensor.defaults.subtensor.register.update_interval) + parser.add_argument('--' + prefix_str + 'subtensor.register.num_processes', '-n', dest=prefix_str + 'subtensor.register.num_processes', help="Number of processors to use for registration", type=int, default=bittensor.defaults.subtensor.register.num_processes) + parser.add_argument('--' + prefix_str + 'subtensor.register.update_interval', '--' + prefix_str + 'subtensor.register.cuda.update_interval', '--' + prefix_str + 'cuda.update_interval', '-u', help="The number of nonces to process before checking for next block during registration", type=int, default=bittensor.defaults.subtensor.register.update_interval) parser.add_argument('--' + prefix_str + 'subtensor.register.no_output_in_place', '--' + prefix_str + 'no_output_in_place', dest="subtensor.register.output_in_place", help="Whether to not ouput the registration statistics in-place. Set flag to disable output in-place.", action='store_false', required=False, default=bittensor.defaults.subtensor.register.output_in_place) parser.add_argument('--' + prefix_str + 'subtensor.register.verbose', help="Whether to ouput the registration statistics verbosely.", action='store_true', required=False, default=bittensor.defaults.subtensor.register.verbose) @@ -169,8 +168,8 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ): parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.use_cuda', '--' + prefix_str + 'cuda', '--' + prefix_str + 'cuda.use_cuda', default=bittensor.defaults.subtensor.register.cuda.use_cuda, help='''Set flag to use CUDA to register.''', action="store_true", required=False ) parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.no_cuda', '--' + prefix_str + 'no_cuda', '--' + prefix_str + 'cuda.no_cuda', dest=prefix_str + 'subtensor.register.cuda.use_cuda', default=not bittensor.defaults.subtensor.register.cuda.use_cuda, help='''Set flag to not use CUDA for registration''', action="store_false", required=False ) - parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.dev_id', '--' + prefix_str + 'cuda.dev_id', type=type_or_suppress(int), nargs='+', default=bittensor.defaults.subtensor.register.cuda.dev_id, help='''Set the CUDA device id(s). Goes by the order of speed. (i.e. 0 is the fastest).''', required=False ) - parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.TPB', '--' + prefix_str + 'cuda.TPB', type=type_or_suppress(int), default=bittensor.defaults.subtensor.register.cuda.TPB, help='''Set the number of Threads Per Block for CUDA.''', required=False ) + parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.dev_id', '--' + prefix_str + 'cuda.dev_id', type=int, nargs='+', default=bittensor.defaults.subtensor.register.cuda.dev_id, help='''Set the CUDA device id(s). Goes by the order of speed. (i.e. 0 is the fastest).''', required=False ) + parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.TPB', '--' + prefix_str + 'cuda.TPB', type=int, default=bittensor.defaults.subtensor.register.cuda.TPB, help='''Set the number of Threads Per Block for CUDA.''', required=False ) except argparse.ArgumentError: # re-parsing arguments. From fedeabe4ae84fa3eb2fb0862c94512bf6001c0e1 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 21 Jun 2023 23:30:13 -0400 Subject: [PATCH 10/40] remove to defaults test --- tests/unit_tests/bittensor_tests/test_config.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/unit_tests/bittensor_tests/test_config.py b/tests/unit_tests/bittensor_tests/test_config.py index c4bae04942..ba2abeb1d4 100644 --- a/tests/unit_tests/bittensor_tests/test_config.py +++ b/tests/unit_tests/bittensor_tests/test_config.py @@ -139,12 +139,7 @@ def construct_config(): return defaults -def test_to_defaults(): - config = construct_config() - config.to_defaults() - if __name__ == "__main__": # test_loaded_config() # test_strict() - # test_to_defaults() test_prefix() \ No newline at end of file From 9bd4a8266e9eb4ff3ca8cb4e136b949c56437a00 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 21 Jun 2023 23:34:30 -0400 Subject: [PATCH 11/40] use dot get for prefix check --- bittensor/_axon/__init__.py | 2 +- bittensor/_dataset/__init__.py | 2 +- bittensor/_logging/__init__.py | 2 +- bittensor/_prometheus/__init__.py | 2 +- bittensor/_subtensor/__init__.py | 2 +- bittensor/_threadpool/__init__.py | 2 +- bittensor/_wallet/__init__.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bittensor/_axon/__init__.py b/bittensor/_axon/__init__.py index 188268caeb..2768f94350 100644 --- a/bittensor/_axon/__init__.py +++ b/bittensor/_axon/__init__.py @@ -154,7 +154,7 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None): """Accept specific arguments from parser""" prefix_str = "" if prefix is None else prefix + "." if prefix is not None: - if not hasattr(bittensor.defaults, prefix): + if bittensor.defaults.get(prefix, d=None) == None: setattr(bittensor.defaults, prefix, bittensor.Config()) getattr(bittensor.defaults, prefix).axon = bittensor.defaults.axon diff --git a/bittensor/_dataset/__init__.py b/bittensor/_dataset/__init__.py index 92628dfb04..b9d0286f00 100644 --- a/bittensor/_dataset/__init__.py +++ b/bittensor/_dataset/__init__.py @@ -135,7 +135,7 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ): """ prefix_str = '' if prefix == None else prefix + '.' if prefix is not None: - if not hasattr(bittensor.defaults, prefix): + if bittensor.defaults.get(prefix, d=None) == None: setattr(bittensor.defaults, prefix, bittensor.Config()) getattr(bittensor.defaults, prefix).dataset = bittensor.defaults.dataset try: diff --git a/bittensor/_logging/__init__.py b/bittensor/_logging/__init__.py index 9ae0b35b02..f6cfe7e80d 100644 --- a/bittensor/_logging/__init__.py +++ b/bittensor/_logging/__init__.py @@ -144,7 +144,7 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ): """ prefix_str = '' if prefix == None else prefix + '.' if prefix is not None: - if not hasattr(bittensor.defaults, prefix): + if bittensor.defaults.get(prefix, d=None) == None: setattr(bittensor.defaults, prefix, bittensor.Config()) getattr(bittensor.defaults, prefix).logging = bittensor.defaults.logging try: diff --git a/bittensor/_prometheus/__init__.py b/bittensor/_prometheus/__init__.py index 9d22c7a573..b84c8b920e 100644 --- a/bittensor/_prometheus/__init__.py +++ b/bittensor/_prometheus/__init__.py @@ -150,7 +150,7 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ): """ prefix_str = '' if prefix == None else prefix + '.' if prefix is not None: - if not hasattr(bittensor.defaults, prefix): + if bittensor.defaults.get(prefix, d=None) == None: setattr(bittensor.defaults, prefix, bittensor.Config()) getattr(bittensor.defaults, prefix).prometheus = bittensor.defaults.prometheus try: diff --git a/bittensor/_subtensor/__init__.py b/bittensor/_subtensor/__init__.py index fa207a5970..1ac4f1943a 100644 --- a/bittensor/_subtensor/__init__.py +++ b/bittensor/_subtensor/__init__.py @@ -142,7 +142,7 @@ def help(cls): def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ): prefix_str = '' if prefix == None else prefix + '.' if prefix is not None: - if not hasattr(bittensor.defaults, prefix): + if bittensor.defaults.get(prefix, d=None) == None: setattr(bittensor.defaults, prefix, bittensor.Config()) getattr(bittensor.defaults, prefix).subtensor = bittensor.defaults.subtensor try: diff --git a/bittensor/_threadpool/__init__.py b/bittensor/_threadpool/__init__.py index ac76d4812d..fe47e14a4a 100644 --- a/bittensor/_threadpool/__init__.py +++ b/bittensor/_threadpool/__init__.py @@ -56,7 +56,7 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ): """ prefix_str = '' if prefix == None else prefix + '.' if prefix is not None: - if not hasattr(bittensor.defaults, prefix): + if bittensor.defaults.get(prefix, d=None) == None: setattr(bittensor.defaults, prefix, bittensor.Config()) getattr(bittensor.defaults, prefix).priority = bittensor.defaults.priority try: diff --git a/bittensor/_wallet/__init__.py b/bittensor/_wallet/__init__.py index 79422c885e..f1e9787c40 100644 --- a/bittensor/_wallet/__init__.py +++ b/bittensor/_wallet/__init__.py @@ -107,7 +107,7 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ): """ prefix_str = '' if prefix == None else prefix + '.' if prefix is not None: - if not hasattr(bittensor.defaults, prefix): + if bittensor.defaults.get(prefix, d=None) == None: setattr(bittensor.defaults, prefix, bittensor.Config()) getattr(bittensor.defaults, prefix).wallet = bittensor.defaults.wallet try: From 98836b4a35f1ebf2ae8195550c5718c331684468 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 01:09:58 -0400 Subject: [PATCH 12/40] move test to package --- .../unit_tests/bittensor_tests/test_wallet.py | 74 ------------------- 1 file changed, 74 deletions(-) diff --git a/tests/unit_tests/bittensor_tests/test_wallet.py b/tests/unit_tests/bittensor_tests/test_wallet.py index f090131bcc..b721122dd8 100644 --- a/tests/unit_tests/bittensor_tests/test_wallet.py +++ b/tests/unit_tests/bittensor_tests/test_wallet.py @@ -20,80 +20,6 @@ import pytest import bittensor -class TestWallet(unittest.TestCase): - def setUp(self): - self.mock_wallet = bittensor.wallet( _mock = True ) - - def test_regen_coldkeypub_from_ss58_addr(self): - ss58_address = "5DD26kC2kxajmwfbbZmVmxhrY9VeeyR1Gpzy9i8wxLUg6zxm" - with patch.object(self.mock_wallet, 'set_coldkeypub') as mock_set_coldkeypub: - self.mock_wallet.regenerate_coldkeypub( ss58_address=ss58_address ) - - mock_set_coldkeypub.assert_called_once() - keypair: bittensor.Keypair = mock_set_coldkeypub.call_args_list[0][0][0] - self.assertEqual(keypair.ss58_address, ss58_address) - - ss58_address_bad = "5DD26kC2kxajmwfbbZmVmxhrY9VeeyR1Gpzy9i8wxLUg6zx" # 1 character short - with pytest.raises(ValueError): - self.mock_wallet.regenerate_coldkeypub(ss58_address=ss58_address_bad) - - def test_regen_coldkeypub_from_hex_pubkey_str(self): - pubkey_str = "0x32939b6abc4d81f02dff04d2b8d1d01cc8e71c5e4c7492e4fa6a238cdca3512f" - with patch.object(self.mock_wallet, 'set_coldkeypub') as mock_set_coldkeypub: - self.mock_wallet.regenerate_coldkeypub(public_key=pubkey_str) - - mock_set_coldkeypub.assert_called_once() - keypair: bittensor.Keypair = mock_set_coldkeypub.call_args_list[0][0][0] - self.assertEqual('0x' + keypair.public_key.hex(), pubkey_str) - - pubkey_str_bad = "0x32939b6abc4d81f02dff04d2b8d1d01cc8e71c5e4c7492e4fa6a238cdca3512" # 1 character short - with pytest.raises(ValueError): - self.mock_wallet.regenerate_coldkeypub(ss58_address=pubkey_str_bad) - - def test_regen_coldkeypub_from_hex_pubkey_bytes(self): - pubkey_str = "0x32939b6abc4d81f02dff04d2b8d1d01cc8e71c5e4c7492e4fa6a238cdca3512f" - pubkey_bytes = bytes.fromhex(pubkey_str[2:]) # Remove 0x from beginning - with patch.object(self.mock_wallet, 'set_coldkeypub') as mock_set_coldkeypub: - self.mock_wallet.regenerate_coldkeypub(public_key=pubkey_bytes) - - mock_set_coldkeypub.assert_called_once() - keypair: bittensor.Keypair = mock_set_coldkeypub.call_args_list[0][0][0] - self.assertEqual(keypair.public_key, pubkey_bytes) - - def test_regen_coldkeypub_no_pubkey(self): - with pytest.raises(ValueError): - # Must provide either public_key or ss58_address - self.mock_wallet.regenerate_coldkeypub(ss58_address=None, public_key=None) - - def test_regen_coldkey_from_hex_seed_str(self): - ss58_addr = "5D5cwd8DX6ij7nouVcoxDuWtJfiR1BnzCkiBVTt7DU8ft5Ta" - seed_str = "0x659c024d5be809000d0d93fe378cfde020846150b01c49a201fc2a02041f7636" - with patch.object(self.mock_wallet, 'set_coldkey') as mock_set_coldkey: - self.mock_wallet.regenerate_coldkey(seed=seed_str) - - mock_set_coldkey.assert_called_once() - keypair: bittensor.Keypair = mock_set_coldkey.call_args_list[0][0][0] - self.assertRegex(keypair.seed_hex if isinstance(keypair.seed_hex, str) else keypair.seed_hex.hex(), rf'(0x|){seed_str[2:]}') - self.assertEqual(keypair.ss58_address, ss58_addr) # Check that the ss58 address is correct - - seed_str_bad = "0x659c024d5be809000d0d93fe378cfde020846150b01c49a201fc2a02041f763" # 1 character short - with pytest.raises(ValueError): - self.mock_wallet.regenerate_coldkey(seed=seed_str_bad) - - def test_regen_hotkey_from_hex_seed_str(self): - ss58_addr = "5D5cwd8DX6ij7nouVcoxDuWtJfiR1BnzCkiBVTt7DU8ft5Ta" - seed_str = "0x659c024d5be809000d0d93fe378cfde020846150b01c49a201fc2a02041f7636" - with patch.object(self.mock_wallet, 'set_hotkey') as mock_set_hotkey: - self.mock_wallet.regenerate_hotkey(seed=seed_str) - - mock_set_hotkey.assert_called_once() - keypair: bittensor.Keypair = mock_set_hotkey.call_args_list[0][0][0] - self.assertRegex(keypair.seed_hex if isinstance(keypair.seed_hex, str) else keypair.seed_hex.hex(), rf'(0x|){seed_str[2:]}') - self.assertEqual(keypair.ss58_address, ss58_addr) # Check that the ss58 address is correct - - seed_str_bad = "0x659c024d5be809000d0d93fe378cfde020846150b01c49a201fc2a02041f763" # 1 character short - with pytest.raises(ValueError): - self.mock_wallet.regenerate_hotkey(seed=seed_str_bad) class TestWalletReregister(unittest.TestCase): def test_wallet_reregister_use_cuda_flag_none(self): From 50fec0d6350a0e96071b7cf98e55a7e81ea6e024 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 01:10:21 -0400 Subject: [PATCH 13/40] move wallet mock --- tests/mocks/wallet_mock.py | 59 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 tests/mocks/wallet_mock.py diff --git a/tests/mocks/wallet_mock.py b/tests/mocks/wallet_mock.py new file mode 100644 index 0000000000..0aa4c18a3a --- /dev/null +++ b/tests/mocks/wallet_mock.py @@ -0,0 +1,59 @@ + +import os +import bittensor +import openwallet + +class Wallet_mock(openwallet.Wallet): + """ + Mocked Version of the bittensor wallet class, meant to be used for testing + """ + def __init__( + self, + _mock:bool, + **kwargs, + ): + r""" Init bittensor wallet object containing a hot and coldkey. + Args: + _mock (required=True, default=False): + If true creates a mock wallet with random keys. + """ + super().__init__(**kwargs) + # For mocking. + self._is_mock = _mock + self._mocked_coldkey_keyfile = None + self._mocked_hotkey_keyfile = None + + print("---- MOCKED WALLET INITIALIZED- ---") + + @property + def hotkey_file(self) -> 'bittensor.Keyfile': + if self._is_mock: + if self._mocked_hotkey_keyfile == None: + self._mocked_hotkey_keyfile = bittensor.keyfile(path='MockedHotkey', _mock = True) + return self._mocked_hotkey_keyfile + else: + wallet_path = os.path.expanduser(os.path.join(self.path, self.name)) + hotkey_path = os.path.join(wallet_path, "hotkeys", self.hotkey_str) + return bittensor.keyfile( path = hotkey_path ) + + @property + def coldkey_file(self) -> 'bittensor.Keyfile': + if self._is_mock: + if self._mocked_coldkey_keyfile == None: + self._mocked_coldkey_keyfile = bittensor.keyfile(path='MockedColdkey', _mock = True) + return self._mocked_coldkey_keyfile + else: + wallet_path = os.path.expanduser(os.path.join(self.path, self.name)) + coldkey_path = os.path.join(wallet_path, "coldkey") + return bittensor.keyfile( path = coldkey_path ) + + @property + def coldkeypub_file(self) -> 'bittensor.Keyfile': + if self._is_mock: + if self._mocked_coldkey_keyfile == None: + self._mocked_coldkey_keyfile = bittensor.keyfile(path='MockedColdkeyPub', _mock = True) + return self._mocked_coldkey_keyfile + else: + wallet_path = os.path.expanduser(os.path.join(self.path, self.name)) + coldkeypub_path = os.path.join(wallet_path, "coldkeypub.txt") + return bittensor.Keyfile( path = coldkeypub_path ) \ No newline at end of file From 918cf4aa1158eedf03a1164679edda5de1a8c097 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 01:10:35 -0400 Subject: [PATCH 14/40] move some utils to package --- bittensor/utils/__init__.py | 77 ++----------------------------------- 1 file changed, 3 insertions(+), 74 deletions(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index ed8d7db31f..42f313389e 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -1,5 +1,5 @@ import numbers -from typing import Callable, Union, List, Optional, Dict, Literal, Type, Any +from typing import Callable, Union, List, Optional, Dict, Literal, Any import bittensor import pandas @@ -7,8 +7,9 @@ import torch import scalecodec import argparse -from substrateinterface import Keypair from substrateinterface.utils import ss58 +from openwallet.utils import * + from .registration import create_pow RAOPERTAO = 1e9 @@ -73,78 +74,6 @@ def version_checking(): if latest_version_as_int > bittensor.__version_as_int__: print('\u001b[33mBittensor Version: Current {}/Latest {}\nPlease update to the latest version at your earliest convenience\u001b[0m'.format(bittensor.__version__,latest_version)) -def is_valid_ss58_address( address: str ) -> bool: - """ - Checks if the given address is a valid ss58 address. - - Args: - address(str): The address to check. - - Returns: - True if the address is a valid ss58 address for Bittensor, False otherwise. - """ - try: - return ss58.is_valid_ss58_address( address, valid_ss58_format=bittensor.__ss58_format__ ) or \ - ss58.is_valid_ss58_address( address, valid_ss58_format=42 ) # Default substrate ss58 format (legacy) - except (IndexError): - return False - -def is_valid_ed25519_pubkey( public_key: Union[str, bytes] ) -> bool: - """ - Checks if the given public_key is a valid ed25519 key. - - Args: - public_key(Union[str, bytes]): The public_key to check. - - Returns: - True if the public_key is a valid ed25519 key, False otherwise. - - """ - try: - if isinstance( public_key, str ): - if len(public_key) != 64 and len(public_key) != 66: - raise ValueError( "a public_key should be 64 or 66 characters" ) - elif isinstance( public_key, bytes ): - if len(public_key) != 32: - raise ValueError( "a public_key should be 32 bytes" ) - else: - raise ValueError( "public_key must be a string or bytes" ) - - keypair = Keypair( - public_key=public_key, - ss58_format=bittensor.__ss58_format__ - ) - - ss58_addr = keypair.ss58_address - return ss58_addr is not None - - except (ValueError, IndexError): - return False - -def is_valid_bittensor_address_or_public_key( address: Union[str, bytes] ) -> bool: - """ - Checks if the given address is a valid destination address. - - Args: - address(Union[str, bytes]): The address to check. - - Returns: - True if the address is a valid destination address, False otherwise. - """ - if isinstance( address, str ): - # Check if ed25519 - if address.startswith('0x'): - return is_valid_ed25519_pubkey( address ) - else: - # Assume ss58 address - return is_valid_ss58_address( address ) - elif isinstance( address, bytes ): - # Check if ed25519 - return is_valid_ed25519_pubkey( address ) - else: - # Invalid address type - return False - def get_ss58_format( ss58_address: str ) -> int: """Returns the ss58 format of the given ss58 address.""" return ss58.get_ss58_format( ss58_address ) From 6378eb2e46ebeaaa22ad51fa706f376ceb54c5a2 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 01:10:53 -0400 Subject: [PATCH 15/40] move wallet to package --- bittensor/__init__.py | 23 +- bittensor/_wallet/__init__.py | 144 ----- bittensor/_wallet/wallet_impl.py | 872 ------------------------------- bittensor/_wallet/wallet_mock.py | 59 --- 4 files changed, 8 insertions(+), 1090 deletions(-) delete mode 100644 bittensor/_wallet/__init__.py delete mode 100644 bittensor/_wallet/wallet_impl.py delete mode 100644 bittensor/_wallet/wallet_mock.py diff --git a/bittensor/__init__.py b/bittensor/__init__.py index 5fcfa788f0..cabe53a064 100644 --- a/bittensor/__init__.py +++ b/bittensor/__init__.py @@ -155,8 +155,8 @@ def turn_console_off(): from bittensor._cli import cli as cli from bittensor._axon import axon as axon from bittensor._axon import axon_info as axon_info -from bittensor._wallet import wallet as wallet -from bittensor._keyfile import keyfile as keyfile +from openwallet import wallet as wallet +from openwallet import keyfile as keyfile from bittensor._metagraph import metagraph as metagraph from bittensor._prometheus import prometheus as prometheus from bittensor._subtensor import subtensor as subtensor @@ -169,10 +169,11 @@ def turn_console_off(): # ---- Classes ----- from bittensor._cli.cli_impl import CLI as CLI -from openconfig.config_impl import Config as Config +from openconfig import Config as Config from bittensor._subtensor.chain_data import DelegateInfo as DelegateInfo -from bittensor._wallet.wallet_impl import Wallet as Wallet -from bittensor._keyfile.keyfile_impl import Keyfile as Keyfile +from openwallet import Wallet as Wallet +from openwallet import Keyfile as Keyfile +from openwallet import Keypair as Keypair from bittensor._subtensor.chain_data import NeuronInfo as NeuronInfo from bittensor._subtensor.chain_data import NeuronInfoLite as NeuronInfoLite from bittensor._subtensor.chain_data import PrometheusInfo as PrometheusInfo @@ -184,7 +185,7 @@ def turn_console_off(): from bittensor._ipfs.ipfs_impl import Ipfs as Ipfs # ---- Errors and Exceptions ----- -from bittensor._keyfile.keyfile_impl import KeyFileError as KeyFileError +from openwallet import KeyFileError as KeyFileError from bittensor._proto.bittensor_pb2 import ForwardTextPromptingRequest from bittensor._proto.bittensor_pb2 import ForwardTextPromptingResponse @@ -210,12 +211,6 @@ def turn_console_off(): from bittensor._neuron.base_prompting_miner import BasePromptingMiner from bittensor._neuron.base_huggingface_miner import HuggingFaceMiner -# ---- Errors and Exceptions ----- -from bittensor._keyfile.keyfile_impl import KeyFileError as KeyFileError - -# ---- Errors and Exceptions ----- -from bittensor._keyfile.keyfile_impl import KeyFileError as KeyFileError - # DEFAULTS defaults = Config() defaults.netuid = 1 @@ -223,12 +218,10 @@ def turn_console_off(): axon.add_defaults( defaults ) prioritythreadpool.add_defaults( defaults ) prometheus.add_defaults( defaults ) -wallet.add_defaults( defaults ) +wallet.add_defaults( defaults, prefix = 'wallet' ) dataset.add_defaults( defaults ) logging.add_defaults( defaults ) -from substrateinterface import Keypair as Keypair - # Logging helpers. def trace(): logging.set_trace(True) diff --git a/bittensor/_wallet/__init__.py b/bittensor/_wallet/__init__.py deleted file mode 100644 index f1e9787c40..0000000000 --- a/bittensor/_wallet/__init__.py +++ /dev/null @@ -1,144 +0,0 @@ -# The MIT License (MIT) -# Copyright © 2021 Yuma Rao - -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the “Software”), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. - -# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -import argparse -import copy -import os - -import bittensor -from bittensor.utils import strtobool, type_or_suppress -from . import wallet_impl, wallet_mock - -class wallet: - """ Create and init wallet that stores hot and coldkey - """ - @classmethod - def mock(cls) -> 'bittensor.Wallet': - return wallet( name='mock' ) - - def __new__( - cls, - config: 'bittensor.Config' = None, - name: str = None, - hotkey: str = None, - path: str = None, - _mock: bool = None - ) -> 'bittensor.Wallet': - r""" Init bittensor wallet object containing a hot and coldkey. - - Args: - config (:obj:`bittensor.Config`, `optional`): - bittensor.wallet.config() - name (required=False, default='default'): - The name of the wallet to unlock for running bittensor - hotkey (required=False, default='default'): - The name of hotkey used to running the miner. - path (required=False, default='~/.bittensor/wallets/'): - The path to your bittensor wallets - _mock (required=False, default=False): - If true creates a mock wallet with random keys. - """ - if config == None: - config = wallet.config() - config = copy.deepcopy( config ) - config.wallet.name = name if name != None else config.wallet.name - config.wallet.hotkey = hotkey if hotkey != None else config.wallet.hotkey - config.wallet.path = path if path != None else config.wallet.path - config.wallet._mock = _mock if _mock != None else config.wallet._mock - wallet.check_config( config ) - # Allows mocking from the command line. - if config.wallet.get('name', bittensor.defaults.wallet.name) == 'mock' or config.wallet._mock: - config.wallet._mock = True - _mock = True - - return wallet_mock.Wallet_mock( - name = config.wallet.get('name', bittensor.defaults.wallet.name), - hotkey = config.wallet.get('hotkey', bittensor.defaults.wallet.hotkey), - path = config.wallet.path, - _mock = True, - config = config - ) - - network = config.get('subtensor.network', bittensor.defaults.subtensor.network) - - # Default to finney. - return wallet_impl.Wallet( - name = config.wallet.get('name', bittensor.defaults.wallet.name), - hotkey = config.wallet.get('hotkey', bittensor.defaults.wallet.hotkey), - path = config.wallet.path, - config = config - ) - - @classmethod - def config(cls) -> 'bittensor.Config': - """ Get config from the argument parser - Return: bittensor.config object - """ - parser = argparse.ArgumentParser() - wallet.add_args( parser ) - return bittensor.config( parser ) - - @classmethod - def help(cls): - """ Print help to stdout - """ - parser = argparse.ArgumentParser() - cls.add_args( parser ) - print (cls.__new__.__doc__) - parser.print_help() - - @classmethod - def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ): - """ Accept specific arguments from parser - """ - prefix_str = '' if prefix == None else prefix + '.' - if prefix is not None: - if bittensor.defaults.get(prefix, d=None) == None: - setattr(bittensor.defaults, prefix, bittensor.Config()) - getattr(bittensor.defaults, prefix).wallet = bittensor.defaults.wallet - try: - parser.add_argument('--' + prefix_str + 'wallet.name', required=False, default=bittensor.defaults.wallet.name, help='''The name of the wallet to unlock for running bittensor (name mock is reserved for mocking this wallet)''') - parser.add_argument('--' + prefix_str + 'wallet.hotkey', required=False, default=bittensor.defaults.wallet.hotkey, help='''The name of wallet's hotkey.''') - parser.add_argument('--' + prefix_str + 'wallet.path', required=False, default=bittensor.defaults.wallet.path, help='''The path to your bittensor wallets''') - parser.add_argument('--' + prefix_str + 'wallet._mock', action='store_true', default=bittensor.defaults.wallet._mock, help='To turn on wallet mocking for testing purposes.') - - parser.add_argument('--' + prefix_str + 'wallet.reregister', required=False, action='store', default=bittensor.defaults.wallet.reregister, type=type_or_suppress(strtobool), help='''Whether to reregister the wallet if it is not already registered.''') - - except argparse.ArgumentError as e: - pass - - @classmethod - def add_defaults(cls, defaults): - """ Adds parser defaults to object from enviroment variables. - """ - defaults.wallet = bittensor.Config() - defaults.wallet.name = os.getenv('BT_WALLET_NAME') if os.getenv('BT_WALLET_NAME') != None else 'default' - defaults.wallet.hotkey = os.getenv('BT_WALLET_HOTKEY') if os.getenv('BT_WALLET_HOTKEY') != None else 'default' - defaults.wallet.path = os.getenv('BT_WALLET_PATH') if os.getenv('BT_WALLET_PATH') != None else '~/.bittensor/wallets/' - defaults.wallet._mock = os.getenv('BT_WALLET_MOCK') if os.getenv('BT_WALLET_MOCK') != None else False - # Defaults for registration - defaults.wallet.reregister = True - - @classmethod - def check_config(cls, config: 'bittensor.Config' ): - """ Check config for wallet name/hotkey/path/hotkeys/sort_by - """ - assert 'wallet' in config - assert isinstance(config.wallet.get('name', bittensor.defaults.wallet.name), str) - assert isinstance(config.wallet.get('hotkey', bittensor.defaults.wallet.hotkey), str ) or config.wallet.get('hotkey', bittensor.defaults.wallet.hotkey) == None - assert isinstance(config.wallet.path, str) - assert isinstance(config.wallet.reregister, bool) diff --git a/bittensor/_wallet/wallet_impl.py b/bittensor/_wallet/wallet_impl.py deleted file mode 100644 index dd82d676fa..0000000000 --- a/bittensor/_wallet/wallet_impl.py +++ /dev/null @@ -1,872 +0,0 @@ -""" Implementation of the wallet class, which manages balances with staking and transfer. Also manages hotkey and coldkey. -""" -# The MIT License (MIT) -# Copyright © 2021 Yuma Rao - -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the “Software”), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. - -# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -import os -import sys -from types import SimpleNamespace -from typing import Optional, Union, List, Tuple, Dict, overload - -import bittensor -from bittensor.utils import is_valid_bittensor_address_or_public_key -from substrateinterface import Keypair -from substrateinterface.base import is_valid_ss58_address -from termcolor import colored - - -def display_mnemonic_msg( keypair : Keypair, key_type : str ): - """ Displaying the mnemonic and warning message to keep mnemonic safe - """ - mnemonic = keypair.mnemonic - mnemonic_green = colored(mnemonic, 'green') - print (colored("\nIMPORTANT: Store this mnemonic in a secure (preferable offline place), as anyone " \ - "who has possesion of this mnemonic can use it to regenerate the key and access your tokens. \n", "red")) - print ("The mnemonic to the new {} is:\n\n{}\n".format(key_type, mnemonic_green)) - print ("You can use the mnemonic to recreate the key in case it gets lost. The command to use to regenerate the key using this mnemonic is:") - print("btcli regen_{} --mnemonic {}".format(key_type, mnemonic)) - print('') - -class Wallet(): - """ - Bittensor wallet maintenance class. Each wallet contains a coldkey and a hotkey. - The coldkey is the user's primary key for holding stake in their wallet - and is the only way that users can access Tao. Coldkeys can hold tokens and should be encrypted on your device. - The coldkey must be used to stake and unstake funds from a running node. The hotkey, on the other hand, is only used - for suscribing and setting weights from running code. Hotkeys are linked to coldkeys through the metagraph. - """ - def __init__( - self, - name:str, - path:str, - hotkey:str, - config: 'bittensor.Config' = None, - ): - r""" Init bittensor wallet object containing a hot and coldkey. - Args: - name (required=True, default='default): - The name of the wallet to unlock for running bittensor - hotkey (required=True, default='default): - The name of hotkey used to running the miner. - path (required=True, default='~/.bittensor/wallets/'): - The path to your bittensor wallets - config (:obj:`bittensor.Config`, `optional`): - bittensor.wallet.config() - """ - self.name = name - self.path = path - self.hotkey_str = hotkey - self._hotkey = None - self._coldkey = None - self._coldkeypub = None - self.config = config - - def __str__(self): - return "Wallet ({}, {}, {})".format(self.name, self.hotkey_str, self.path) - - def __repr__(self): - return self.__str__() - - def neuron(self, netuid: int) -> Optional['bittensor.NeuronInfo']: - return self.get_neuron(netuid=netuid) - - def trust(self, netuid: int) -> Optional[float]: - neuron = self.get_neuron(netuid=netuid) - if neuron is None: - return None - return neuron.trust - - def validator_trust(self, netuid: int) -> Optional[float]: - neuron = self.get_neuron(netuid=netuid) - if neuron is None: - return None - return neuron.validator_trust - - def rank(self, netuid: int) -> Optional[float]: - neuron = self.get_neuron(netuid=netuid) - if neuron is None: - return None - return neuron.rank - - def incentive(self, netuid: int) -> Optional[float]: - neuron = self.get_neuron(netuid=netuid) - if neuron is None: - return None - return neuron.incentive - - def dividends(self, netuid: int) -> Optional[float]: - neuron = self.get_neuron(netuid=netuid) - if neuron is None: - return None - return neuron.dividends - - def consensus(self, netuid: int) -> Optional[float]: - neuron = self.get_neuron(netuid=netuid) - if neuron is None: - return None - return neuron.consensus - - def last_update(self, netuid: int) -> Optional[int]: - neuron = self.get_neuron(netuid=netuid) - if neuron is None: - return None - return neuron.last_update - - def validator_permit(self, netuid: int) -> Optional[bool]: - neuron = self.get_neuron(netuid=netuid) - if neuron is None: - return None - return neuron.validator_permit - - def weights(self, netuid: int) -> Optional[List[List[int]]]: - neuron = self.get_neuron(netuid=netuid) - if neuron is None: - return None - return neuron.weights - - def bonds(self, netuid: int) -> Optional[List[List[int]]]: - neuron = self.get_neuron(netuid=netuid) - if neuron is None: - return None - return neuron.bonds - - def uid(self, netuid: int) -> int: - return self.get_uid(netuid=netuid) - - @property - def stake(self) -> 'bittensor.Balance': - return self.get_stake() - - @property - def balance(self) -> 'bittensor.Balance': - return self.get_balance() - - def is_registered( self, subtensor: Optional['bittensor.Subtensor'] = None, netuid: Optional[int] = None ) -> bool: - """ Returns true if this wallet is registered. - Args: - subtensor( Optional['bittensor.Subtensor'] ): - Bittensor subtensor connection. Overrides with defaults if None. - Determines which network we check for registration. - netuid ( Optional[int] ): - The network uid to check for registration. - Default is None, which checks any subnetwork. - Return: - is_registered (bool): - Is the wallet registered on the chain. - """ - if subtensor == None: subtensor = bittensor.subtensor(self.config) - - # default to finney - if netuid == None: - return subtensor.is_hotkey_registered_any( self.hotkey.ss58_address ) - else: - return subtensor.is_hotkey_registered_on_subnet( self.hotkey.ss58_address, netuid = netuid ) - - - def get_neuron ( self, netuid: int, subtensor: Optional['bittensor.Subtensor'] = None ) -> Optional['bittensor.NeuronInfo'] : - """ Returns this wallet's neuron information from subtensor. - Args: - netuid (int): - The network uid of the subnet to query. - subtensor( Optional['bittensor.Subtensor'] ): - Bittensor subtensor connection. Overrides with defaults if None. - Return: - neuron (Union[ NeuronInfo, None ]): - neuron account on the chain or None if you are not registered. - """ - if subtensor == None: subtensor = bittensor.subtensor() - if not self.is_registered(netuid = netuid, subtensor=subtensor): - print(colored('This wallet is not registered. Call wallet.register() before this function.','red')) - return None - neuron = subtensor.neuron_for_wallet( self, netuid = netuid ) - return neuron - - def get_uid ( self, netuid: int, subtensor: Optional['bittensor.Subtensor'] = None ) -> int: - """ Returns this wallet's hotkey uid or -1 if the hotkey is not subscribed. - Args: - netuid (int): - The network uid of the subnet to query. - subtensor( Optional['bittensor.Subtensor'] ): - Bittensor subtensor connection. Overrides with defaults if None. - Return: - uid (int): - Network uid or -1 if you are not registered. - """ - if subtensor == None: subtensor = bittensor.subtensor() - if not self.is_registered(netuid = netuid, subtensor=subtensor): - print(colored('This wallet is not registered. Call wallet.register() before this function.','red')) - return -1 - neuron = self.get_neuron(netuid = netuid, subtensor = subtensor) - if neuron.is_null: - return -1 - else: - return neuron.uid - - def get_stake ( self, subtensor: Optional['bittensor.Subtensor'] = None ) -> 'bittensor.Balance': - """ Returns this wallet's staking balance from passed subtensor connection. - Args: - subtensor( Optional['bittensor.Subtensor'] ): - Bittensor subtensor connection. Overrides with defaults if None. - Return: - balance (bittensor.utils.balance.Balance): - Stake account balance. - """ - if subtensor == None: subtensor = bittensor.subtensor() - stake = subtensor.get_stake_for_coldkey_and_hotkey( hotkey_ss58 = self.hotkey.ss58_address, coldkey_ss58 = self.coldkeypub.ss58_address ) - if not stake: # Not registered. - print(colored('This wallet is not registered. Call wallet.register() before this function.','red')) - return bittensor.Balance(0) - - return stake - - def get_balance( self, subtensor: Optional['bittensor.Subtensor'] = None ) -> 'bittensor.Balance': - """ Returns this wallet's coldkey balance from passed subtensor connection. - Args: - subtensor( Optional['bittensor.Subtensor'] ): - Bittensor subtensor connection. Overrides with defaults if None. - Return: - balance (bittensor.utils.balance.Balance): - Coldkey balance. - """ - if subtensor == None: subtensor = bittensor.subtensor() - return subtensor.get_balance(address = self.coldkeypub.ss58_address) - - def reregister( - self, - netuid: int, - subtensor: Optional['bittensor.Subtensor'] = None, - wait_for_inclusion: bool = False, - wait_for_finalization: bool = True, - prompt: bool = False - ) -> Optional['bittensor.Wallet']: - """ Re-register this wallet on the chain. - Args: - netuid (int): - The network uid of the subnet to register on. - subtensor( Optional['bittensor.Subtensor'] ): - Bittensor subtensor connection. Overrides with defaults if None. - wait_for_inclusion (bool): - if set, waits for the extrinsic to enter a block before returning true, - or returns false if the extrinsic fails to enter the block within the timeout. - wait_for_finalization (bool): - if set, waits for the extrinsic to be finalized on the chain before returning true, - or returns false if the extrinsic fails to be finalized within the timeout. - prompt (bool): - If true, the call waits for confirmation from the user before proceeding. - - Return: - wallet (bittensor.Wallet): - This wallet. - """ - if subtensor == None: - subtensor = bittensor.subtensor() - if not self.is_registered(netuid = netuid, subtensor=subtensor): - # Check if the wallet should reregister - if not self.config.wallet.get('reregister'): - sys.exit(0) - - self.register( - subtensor = subtensor, - netuid = netuid, - prompt = prompt, - TPB = self.config.subtensor.register.cuda.get('TPB', None), - update_interval = self.config.subtensor.register.cuda.get('update_interval', None), - num_processes = self.config.subtensor.register.get('num_processes', None), - cuda = self.config.subtensor.register.cuda.get('use_cuda', bittensor.defaults.subtensor.register.cuda.use_cuda), - dev_id = self.config.subtensor.register.cuda.get('dev_id', None), - wait_for_inclusion = wait_for_inclusion, - wait_for_finalization = wait_for_finalization, - output_in_place = self.config.subtensor.register.get('output_in_place', bittensor.defaults.subtensor.register.output_in_place), - log_verbose = self.config.subtensor.register.get('verbose', bittensor.defaults.subtensor.register.verbose), - ) - - return self - - def register ( - self, - netuid: int, - subtensor: Optional['bittensor.Subtensor'] = None, - wait_for_inclusion: bool = False, - wait_for_finalization: bool = True, - prompt: bool = False, - max_allowed_attempts: int = 3, - cuda: bool = False, - dev_id: int = 0, - TPB: int = 256, - num_processes: Optional[int] = None, - update_interval: Optional[int] = None, - output_in_place: bool = True, - log_verbose: bool = False, - ) -> 'bittensor.Wallet': - """ Registers the wallet to chain. - Args: - netuid (int): - The network uid of the subnet to register on. - subtensor( Optional['bittensor.Subtensor'] ): - Bittensor subtensor connection. Overrides with defaults if None. - wait_for_inclusion (bool): - If set, waits for the extrinsic to enter a block before returning true, - or returns false if the extrinsic fails to enter the block within the timeout. - wait_for_finalization (bool): - If set, waits for the extrinsic to be finalized on the chain before returning true, - or returns false if the extrinsic fails to be finalized within the timeout. - prompt (bool): - If true, the call waits for confirmation from the user before proceeding. - max_allowed_attempts (int): - Maximum number of attempts to register the wallet. - cuda (bool): - If true, the wallet should be registered on the cuda device. - dev_id (int): - The cuda device id. - TPB (int): - The number of threads per block (cuda). - num_processes (int): - The number of processes to use to register. - update_interval (int): - The number of nonces to solve between updates. - output_in_place (bool): - If true, the registration output is printed in-place. - log_verbose (bool): - If true, the registration output is more verbose. - Returns: - success (bool): - flag is true if extrinsic was finalized or uncluded in the block. - If we did not wait for finalization / inclusion, the response is true. - """ - # Get chain connection. - if subtensor == None: subtensor = bittensor.subtensor() - subtensor.register( - wallet = self, - wait_for_inclusion = wait_for_inclusion, - wait_for_finalization = wait_for_finalization, - prompt=prompt, max_allowed_attempts=max_allowed_attempts, - output_in_place = output_in_place, - cuda=cuda, - dev_id=dev_id, - TPB=TPB, - num_processes=num_processes, - update_interval=update_interval, - log_verbose=log_verbose, - netuid = netuid, - ) - - return self - - def add_stake( self, - amount: Union[float, bittensor.Balance] = None, - wait_for_inclusion: bool = False, - wait_for_finalization: bool = True, - subtensor: Optional['bittensor.Subtensor'] = None, - prompt: bool = False - ) -> bool: - """ Stakes tokens from this wallet's coldkey onto it's hotkey. - Args: - amount_tao (float): - amount of tao to stake or bittensor balance object. If None, stakes all available balance. - wait_for_inclusion (bool): - if set, waits for the extrinsic to enter a block before returning true, - or returns false if the extrinsic fails to enter the block within the timeout. - wait_for_finalization (bool): - if set, waits for the extrinsic to be finalized on the chain before returning true, - or returns false if the extrinsic fails to be finalized within the timeout. - subtensor( `bittensor.Subtensor` ): - Bittensor subtensor connection. Overrides with defaults if None. - prompt (bool): - If true, the call waits for confirmation from the user before proceeding. - Returns: - success (bool): - flag is true if extrinsic was finalized or uncluded in the block. - If we did not wait for finalization / inclusion, the response is true. - """ - if subtensor == None: subtensor = bittensor.subtensor() - return subtensor.add_stake( wallet = self, amount = amount, wait_for_inclusion = wait_for_inclusion, wait_for_finalization = wait_for_finalization, prompt=prompt ) - - def remove_stake( self, - amount: Union[float, bittensor.Balance] = None, - wait_for_inclusion: bool = False, - wait_for_finalization: bool = True, - subtensor: Optional['bittensor.Subtensor'] = None, - prompt: bool = False, - ) -> bool: - """ Removes stake from this wallet's hotkey and moves them onto it's coldkey balance. - Args: - amount_tao (float): - amount of tao to unstake or bittensor balance object. If None, unstakes all available hotkey balance. - wait_for_inclusion (bool): - if set, waits for the extrinsic to enter a block before returning true, - or returns false if the extrinsic fails to enter the block within the timeout. - wait_for_finalization (bool): - if set, waits for the extrinsic to be finalized on the chain before returning true, - or returns false if the extrinsic fails to be finalized within the timeout. - subtensor( `bittensor.Subtensor` ): - Bittensor subtensor connection. Overrides with defaults if None. - prompt (bool): - If true, the call waits for confirmation from the user before proceeding. - Returns: - success (bool): - flag is true if extrinsic was finalized or uncluded in the block. - If we did not wait for finalization / inclusion, the response is true. - """ - if subtensor == None: subtensor = bittensor.subtensor() - return subtensor.unstake( wallet = self, amount = amount, wait_for_inclusion = wait_for_inclusion, wait_for_finalization = wait_for_finalization, prompt=prompt ) - - def transfer( - self, - dest:str, - amount: Union[float, bittensor.Balance] , - wait_for_inclusion: bool = False, - wait_for_finalization: bool = True, - subtensor: Optional['bittensor.Subtensor'] = None, - prompt: bool = False, - ) -> bool: - """ Transfers Tao from this wallet's coldkey to the destination address. - Args: - dest (`type`:str, required): - The destination address either encoded as a ss58 or ed255 public-key string of - secondary account. - amount (float, required): - amount of tao to transfer or a bittensor balance object. - wait_for_inclusion (bool): - if set, waits for the extrinsic to enter a block before returning true, - or returns false if the extrinsic fails to enter the block within the timeout. - wait_for_finalization (bool): - if set, waits for the extrinsic to be finalized on the chain before returning true, - or returns false if the extrinsic fails to be finalized within the timeout. - subtensor( `bittensor.Subtensor` ): - Bittensor subtensor connection. Overrides with defaults if None. - prompt (bool): - If true, the call waits for confirmation from the user before proceeding. - Returns: - success (bool): - flag is true if extrinsic was finalized or uncluded in the block. - If we did not wait for finalization / inclusion, the response is true. - """ - if subtensor == None: subtensor = bittensor.subtensor() - return subtensor.transfer( wallet = self, dest = dest, amount = amount, wait_for_inclusion = wait_for_inclusion, wait_for_finalization = wait_for_finalization, prompt=prompt ) - - def create_if_non_existent( self, coldkey_use_password:bool = True, hotkey_use_password:bool = False) -> 'Wallet': - """ Checks for existing coldkeypub and hotkeys and creates them if non-existent. - """ - return self.create(coldkey_use_password, hotkey_use_password) - - def create (self, coldkey_use_password:bool = True, hotkey_use_password:bool = False ) -> 'Wallet': - """ Checks for existing coldkeypub and hotkeys and creates them if non-existent. - """ - # ---- Setup Wallet. ---- - if not self.coldkey_file.exists_on_device() and not self.coldkeypub_file.exists_on_device(): - self.create_new_coldkey( n_words = 12, use_password = coldkey_use_password ) - if not self.hotkey_file.exists_on_device(): - self.create_new_hotkey( n_words = 12, use_password = hotkey_use_password ) - return self - - def recreate (self, coldkey_use_password:bool = True, hotkey_use_password:bool = False ) -> 'Wallet': - """ Checks for existing coldkeypub and hotkeys and creates them if non-existent. - """ - # ---- Setup Wallet. ---- - self.create_new_coldkey( n_words = 12, use_password = coldkey_use_password ) - self.create_new_hotkey( n_words = 12, use_password = hotkey_use_password ) - return self - - @property - def hotkey_file(self) -> 'bittensor.Keyfile': - - wallet_path = os.path.expanduser(os.path.join(self.path, self.name)) - hotkey_path = os.path.join(wallet_path, "hotkeys", self.hotkey_str) - return bittensor.keyfile( path = hotkey_path ) - - @property - def coldkey_file(self) -> 'bittensor.Keyfile': - wallet_path = os.path.expanduser(os.path.join(self.path, self.name)) - coldkey_path = os.path.join(wallet_path, "coldkey") - return bittensor.keyfile( path = coldkey_path ) - - @property - def coldkeypub_file(self) -> 'bittensor.Keyfile': - wallet_path = os.path.expanduser(os.path.join(self.path, self.name)) - coldkeypub_path = os.path.join(wallet_path, "coldkeypub.txt") - return bittensor.Keyfile( path = coldkeypub_path ) - - def set_hotkey(self, keypair: 'bittensor.Keypair', encrypt: bool = False, overwrite: bool = False) -> 'bittensor.Keyfile': - self._hotkey = keypair - self.hotkey_file.set_keypair( keypair, encrypt = encrypt, overwrite = overwrite ) - - def set_coldkeypub(self, keypair: 'bittensor.Keypair', encrypt: bool = False, overwrite: bool = False) -> 'bittensor.Keyfile': - self._coldkeypub = Keypair(ss58_address=keypair.ss58_address) - self.coldkeypub_file.set_keypair( self._coldkeypub, encrypt = encrypt, overwrite = overwrite ) - - def set_coldkey(self, keypair: 'bittensor.Keypair', encrypt: bool = True, overwrite: bool = False) -> 'bittensor.Keyfile': - self._coldkey = keypair - self.coldkey_file.set_keypair( self._coldkey, encrypt = encrypt, overwrite = overwrite ) - - def get_coldkey(self, password: str = None ) -> 'bittensor.Keypair': - self.coldkey_file.get_keypair( password = password ) - - def get_hotkey(self, password: str = None ) -> 'bittensor.Keypair': - self.hotkey_file.get_keypair( password = password ) - - def get_coldkeypub(self, password: str = None ) -> 'bittensor.Keypair': - self.coldkeypub_file.get_keypair( password = password ) - - @property - def hotkey(self) -> 'bittensor.Keypair': - r""" Loads the hotkey from wallet.path/wallet.name/hotkeys/wallet.hotkey or raises an error. - Returns: - hotkey (Keypair): - hotkey loaded from config arguments. - Raises: - KeyFileError: Raised if the file is corrupt of non-existent. - CryptoKeyError: Raised if the user enters an incorrec password for an encrypted keyfile. - """ - if self._hotkey == None: - self._hotkey = self.hotkey_file.keypair - return self._hotkey - - @property - def coldkey(self) -> 'bittensor.Keypair': - r""" Loads the hotkey from wallet.path/wallet.name/coldkey or raises an error. - Returns: - coldkey (Keypair): - colkey loaded from config arguments. - Raises: - KeyFileError: Raised if the file is corrupt of non-existent. - CryptoKeyError: Raised if the user enters an incorrec password for an encrypted keyfile. - """ - if self._coldkey == None: - self._coldkey = self.coldkey_file.keypair - return self._coldkey - - @property - def coldkeypub(self) -> 'bittensor.Keypair': - r""" Loads the coldkeypub from wallet.path/wallet.name/coldkeypub.txt or raises an error. - Returns: - coldkeypub (Keypair): - colkeypub loaded from config arguments. - Raises: - KeyFileError: Raised if the file is corrupt of non-existent. - CryptoKeyError: Raised if the user enters an incorrect password for an encrypted keyfile. - """ - if self._coldkeypub == None: - self._coldkeypub = self.coldkeypub_file.keypair - return self._coldkeypub - - def create_coldkey_from_uri(self, uri:str, use_password: bool = True, overwrite:bool = False) -> 'Wallet': - """ Creates coldkey from suri string, optionally encrypts it with the user's inputed password. - Args: - uri: (str, required): - URI string to use i.e. /Alice or /Bob - use_password (bool, optional): - Is the created key password protected. - overwrite (bool, optional): - Will this operation overwrite the coldkey under the same path //coldkey - Returns: - wallet (bittensor.Wallet): - this object with newly created coldkey. - """ - keypair = Keypair.create_from_uri( uri ) - display_mnemonic_msg( keypair, "coldkey" ) - self.set_coldkey( keypair, encrypt = use_password, overwrite = overwrite) - self.set_coldkeypub( keypair, overwrite = overwrite) - return self - - def create_hotkey_from_uri( self, uri:str, use_password: bool = False, overwrite:bool = False) -> 'Wallet': - """ Creates hotkey from suri string, optionally encrypts it with the user's inputed password. - Args: - uri: (str, required): - URI string to use i.e. /Alice or /Bob - use_password (bool, optional): - Is the created key password protected. - overwrite (bool, optional): - Will this operation overwrite the hotkey under the same path //hotkeys/ - Returns: - wallet (bittensor.Wallet): - this object with newly created hotkey. - """ - keypair = Keypair.create_from_uri( uri ) - display_mnemonic_msg( keypair, "hotkey" ) - self.set_hotkey( keypair, encrypt=use_password, overwrite = overwrite) - return self - - def new_coldkey( self, n_words:int = 12, use_password: bool = True, overwrite:bool = False) -> 'Wallet': - """ Creates a new coldkey, optionally encrypts it with the user's inputed password and saves to disk. - Args: - n_words: (int, optional): - Number of mnemonic words to use. - use_password (bool, optional): - Is the created key password protected. - overwrite (bool, optional): - Will this operation overwrite the coldkey under the same path //coldkey - Returns: - wallet (bittensor.Wallet): - this object with newly created coldkey. - """ - self.create_new_coldkey( n_words, use_password, overwrite ) - - def create_new_coldkey( self, n_words:int = 12, use_password: bool = True, overwrite:bool = False) -> 'Wallet': - """ Creates a new coldkey, optionally encrypts it with the user's inputed password and saves to disk. - Args: - n_words: (int, optional): - Number of mnemonic words to use. - use_password (bool, optional): - Is the created key password protected. - overwrite (bool, optional): - Will this operation overwrite the coldkey under the same path //coldkey - Returns: - wallet (bittensor.Wallet): - this object with newly created coldkey. - """ - mnemonic = Keypair.generate_mnemonic( n_words) - keypair = Keypair.create_from_mnemonic(mnemonic) - display_mnemonic_msg( keypair, "coldkey" ) - self.set_coldkey( keypair, encrypt = use_password, overwrite = overwrite) - self.set_coldkeypub( keypair, overwrite = overwrite) - return self - - def new_hotkey( self, n_words:int = 12, use_password: bool = False, overwrite:bool = False) -> 'Wallet': - """ Creates a new hotkey, optionally encrypts it with the user's inputed password and saves to disk. - Args: - n_words: (int, optional): - Number of mnemonic words to use. - use_password (bool, optional): - Is the created key password protected. - overwrite (bool, optional): - Will this operation overwrite the hotkey under the same path //hotkeys/ - Returns: - wallet (bittensor.Wallet): - this object with newly created hotkey. - """ - self.create_new_hotkey( n_words, use_password, overwrite ) - - def create_new_hotkey( self, n_words:int = 12, use_password: bool = False, overwrite:bool = False) -> 'Wallet': - """ Creates a new hotkey, optionally encrypts it with the user's inputed password and saves to disk. - Args: - n_words: (int, optional): - Number of mnemonic words to use. - use_password (bool, optional): - Is the created key password protected. - overwrite (bool, optional): - Will this operation overwrite the hotkey under the same path //hotkeys/ - Returns: - wallet (bittensor.Wallet): - this object with newly created hotkey. - """ - mnemonic = Keypair.generate_mnemonic( n_words) - keypair = Keypair.create_from_mnemonic(mnemonic) - display_mnemonic_msg( keypair, "hotkey" ) - self.set_hotkey( keypair, encrypt=use_password, overwrite = overwrite) - return self - - def regenerate_coldkeypub( self, ss58_address: Optional[str] = None, public_key: Optional[Union[str, bytes]] = None, overwrite: bool = False ) -> 'Wallet': - """ Regenerates the coldkeypub from passed ss58_address or public_key and saves the file - Requires either ss58_address or public_key to be passed. - Args: - ss58_address: (str, optional): - Address as ss58 string. - public_key: (str | bytes, optional): - Public key as hex string or bytes. - overwrite (bool, optional) (default: False): - Will this operation overwrite the coldkeypub (if exists) under the same path //coldkeypub - Returns: - wallet (bittensor.Wallet): - newly re-generated Wallet with coldkeypub. - - """ - if ss58_address is None and public_key is None: - raise ValueError("Either ss58_address or public_key must be passed") - - if not is_valid_bittensor_address_or_public_key( ss58_address if ss58_address is not None else public_key ): - raise ValueError(f"Invalid {'ss58_address' if ss58_address is not None else 'public_key'}") - - if ss58_address is not None: - ss58_format = bittensor.utils.get_ss58_format( ss58_address ) - keypair = Keypair(ss58_address=ss58_address, public_key=public_key, ss58_format=ss58_format) - else: - keypair = Keypair(ss58_address=ss58_address, public_key=public_key, ss58_format=bittensor.__ss58_format__) - - # No need to encrypt the public key - self.set_coldkeypub( keypair, overwrite = overwrite) - - return self - - # Short name for regenerate_coldkeypub - regen_coldkeypub = regenerate_coldkeypub - - @overload - def regenerate_coldkey( - self, - mnemonic: Optional[Union[list, str]] = None, - use_password: bool = True, - overwrite: bool = False - ) -> 'Wallet': - ... - - @overload - def regenerate_coldkey( - self, - seed: Optional[str] = None, - use_password: bool = True, - overwrite: bool = False - ) -> 'Wallet': - ... - - @overload - def regenerate_coldkey( - self, - json: Optional[Tuple[Union[str, Dict], str]] = None, - use_password: bool = True, - overwrite: bool = False - ) -> 'Wallet': - ... - - - def regenerate_coldkey( - self, - use_password: bool = True, - overwrite: bool = False, - **kwargs - ) -> 'Wallet': - """ Regenerates the coldkey from passed mnemonic, seed, or json encrypts it with the user's password and saves the file - Args: - mnemonic: (Union[list, str], optional): - Key mnemonic as list of words or string space separated words. - seed: (str, optional): - Seed as hex string. - json: (Tuple[Union[str, Dict], str], optional): - Restore from encrypted JSON backup as (json_data: Union[str, Dict], passphrase: str) - use_password (bool, optional): - Is the created key password protected. - overwrite (bool, optional): - Will this operation overwrite the coldkey under the same path //coldkey - Returns: - wallet (bittensor.Wallet): - this object with newly created coldkey. - - Note: uses priority order: mnemonic > seed > json - """ - if len(kwargs) == 0: - raise ValueError("Must pass either mnemonic, seed, or json") - - # Get from kwargs - mnemonic = kwargs.get('mnemonic', None) - seed = kwargs.get('seed', None) - json = kwargs.get('json', None) - - if mnemonic is None and seed is None and json is None: - raise ValueError("Must pass either mnemonic, seed, or json") - if mnemonic is not None: - if isinstance( mnemonic, str): mnemonic = mnemonic.split() - if len(mnemonic) not in [12,15,18,21,24]: - raise ValueError("Mnemonic has invalid size. This should be 12,15,18,21 or 24 words") - keypair = Keypair.create_from_mnemonic(" ".join(mnemonic), ss58_format=bittensor.__ss58_format__ ) - display_mnemonic_msg( keypair, "coldkey" ) - elif seed is not None: - keypair = Keypair.create_from_seed(seed, ss58_format=bittensor.__ss58_format__ ) - else: - # json is not None - if not isinstance(json, tuple) or len(json) != 2 or not isinstance(json[0], (str, dict)) or not isinstance(json[1], str): - raise ValueError("json must be a tuple of (json_data: str | Dict, passphrase: str)") - - json_data, passphrase = json - keypair = Keypair.create_from_encrypted_json( json_data, passphrase, ss58_format=bittensor.__ss58_format__ ) - - self.set_coldkey( keypair, encrypt = use_password, overwrite = overwrite) - self.set_coldkeypub( keypair, overwrite = overwrite) - return self - - # Short name for regenerate_coldkey - regen_coldkey = regenerate_coldkey - - @overload - def regenerate_hotkey( - self, - mnemonic: Optional[Union[list, str]] = None, - use_password: bool = True, - overwrite: bool = False - ) -> 'Wallet': - ... - - @overload - def regenerate_hotkey( - self, - seed: Optional[str] = None, - use_password: bool = True, - overwrite: bool = False - ) -> 'Wallet': - ... - - @overload - def regenerate_hotkey( - self, - json: Optional[Tuple[Union[str, Dict], str]] = None, - use_password: bool = True, - overwrite: bool = False - ) -> 'Wallet': - ... - - def regenerate_hotkey( - self, - use_password: bool = True, - overwrite: bool = False, - **kwargs - ) -> 'Wallet': - """ Regenerates the hotkey from passed mnemonic, encrypts it with the user's password and save the file - Args: - mnemonic: (Union[list, str], optional): - Key mnemonic as list of words or string space separated words. - seed: (str, optional): - Seed as hex string. - json: (Tuple[Union[str, Dict], str], optional): - Restore from encrypted JSON backup as (json_data: Union[str, Dict], passphrase: str) - use_password (bool, optional): - Is the created key password protected. - overwrite (bool, optional): - Will this operation overwrite the hotkey under the same path //hotkeys/ - Returns: - wallet (bittensor.Wallet): - this object with newly created hotkey. - """ - if len(kwargs) == 0: - raise ValueError("Must pass either mnemonic, seed, or json") - - # Get from kwargs - mnemonic = kwargs.get('mnemonic', None) - seed = kwargs.get('seed', None) - json = kwargs.get('json', None) - - if mnemonic is None and seed is None and json is None: - raise ValueError("Must pass either mnemonic, seed, or json") - if mnemonic is not None: - if isinstance( mnemonic, str): mnemonic = mnemonic.split() - if len(mnemonic) not in [12,15,18,21,24]: - raise ValueError("Mnemonic has invalid size. This should be 12,15,18,21 or 24 words") - keypair = Keypair.create_from_mnemonic(" ".join(mnemonic), ss58_format=bittensor.__ss58_format__ ) - display_mnemonic_msg( keypair, "hotkey" ) - elif seed is not None: - keypair = Keypair.create_from_seed(seed, ss58_format=bittensor.__ss58_format__ ) - else: - # json is not None - if not isinstance(json, tuple) or len(json) != 2 or not isinstance(json[0], (str, dict)) or not isinstance(json[1], str): - raise ValueError("json must be a tuple of (json_data: str | Dict, passphrase: str)") - - json_data, passphrase = json - keypair = Keypair.create_from_encrypted_json( json_data, passphrase, ss58_format=bittensor.__ss58_format__ ) - - - self.set_hotkey( keypair, encrypt=use_password, overwrite = overwrite) - return self - - # Short name for regenerate_hotkey - regen_hotkey = regenerate_hotkey diff --git a/bittensor/_wallet/wallet_mock.py b/bittensor/_wallet/wallet_mock.py deleted file mode 100644 index f2720fcae3..0000000000 --- a/bittensor/_wallet/wallet_mock.py +++ /dev/null @@ -1,59 +0,0 @@ - -from . import wallet_impl -import os -import bittensor - -class Wallet_mock(wallet_impl.Wallet): - """ - Mocked Version of the bittensor wallet class, meant to be used for testing - """ - def __init__( - self, - _mock:bool, - **kwargs, - ): - r""" Init bittensor wallet object containing a hot and coldkey. - Args: - _mock (required=True, default=False): - If true creates a mock wallet with random keys. - """ - super().__init__(**kwargs) - # For mocking. - self._is_mock = _mock - self._mocked_coldkey_keyfile = None - self._mocked_hotkey_keyfile = None - - print("---- MOCKED WALLET INITIALIZED- ---") - - @property - def hotkey_file(self) -> 'bittensor.Keyfile': - if self._is_mock: - if self._mocked_hotkey_keyfile == None: - self._mocked_hotkey_keyfile = bittensor.keyfile(path='MockedHotkey', _mock = True) - return self._mocked_hotkey_keyfile - else: - wallet_path = os.path.expanduser(os.path.join(self.path, self.name)) - hotkey_path = os.path.join(wallet_path, "hotkeys", self.hotkey_str) - return bittensor.keyfile( path = hotkey_path ) - - @property - def coldkey_file(self) -> 'bittensor.Keyfile': - if self._is_mock: - if self._mocked_coldkey_keyfile == None: - self._mocked_coldkey_keyfile = bittensor.keyfile(path='MockedColdkey', _mock = True) - return self._mocked_coldkey_keyfile - else: - wallet_path = os.path.expanduser(os.path.join(self.path, self.name)) - coldkey_path = os.path.join(wallet_path, "coldkey") - return bittensor.keyfile( path = coldkey_path ) - - @property - def coldkeypub_file(self) -> 'bittensor.Keyfile': - if self._is_mock: - if self._mocked_coldkey_keyfile == None: - self._mocked_coldkey_keyfile = bittensor.keyfile(path='MockedColdkeyPub', _mock = True) - return self._mocked_coldkey_keyfile - else: - wallet_path = os.path.expanduser(os.path.join(self.path, self.name)) - coldkeypub_path = os.path.join(wallet_path, "coldkeypub.txt") - return bittensor.Keyfile( path = coldkeypub_path ) \ No newline at end of file From 5a66d272e5a9b20b71acce6312cf1e18fbb08d49 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 01:22:50 -0400 Subject: [PATCH 16/40] move reregister to registration utils --- bittensor/utils/registration.py | 74 +++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/bittensor/utils/registration.py b/bittensor/utils/registration.py index 16e92262bf..b5378f7b7a 100644 --- a/bittensor/utils/registration.py +++ b/bittensor/utils/registration.py @@ -4,6 +4,7 @@ import multiprocessing import os import random +import sys import time from dataclasses import dataclass from datetime import timedelta @@ -869,3 +870,76 @@ def create_pow( ) return solution + + +def reregister_wallet( + netuid: int, + wallet: 'bittensor.Wallet', + subtensor: Optional['bittensor.Subtensor'] = None, + config: Optional['bittensor.Config'] = None, + wait_for_inclusion: bool = False, + wait_for_finalization: bool = True, + prompt: bool = False + ) -> Optional['bittensor.Wallet']: + """ Re-register this a Wallet on the chain, or exits. + Exits if the wallet is not registered on the chain AND + the config.subtensor.reregister flag is set to False. + Args: + netuid (int): + The network uid of the subnet to register on. + wallet( 'bittensor.Wallet' ): + Bittensor Wallet to re-register + subtensor (Optional['bittensor.Subtensor']): + Bittensor subtensor to use for registration. + config (Optional['bittensor.Config']): + Bittensor config to use for registration. + wait_for_inclusion (bool): + if set, waits for the extrinsic to enter a block before returning true, + or returns false if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): + if set, waits for the extrinsic to be finalized on the chain before returning true, + or returns false if the extrinsic fails to be finalized within the timeout. + prompt (bool): + If true, the call waits for confirmation from the user before proceeding. + + Return: + wallet (bittensor.Wallet): + This wallet. + + Raises: + SytemExit(0): + If the wallet is not registered on the chain AND + the config.subtensor.reregister flag is set to False. + """ + wallet.hotkey + + if config is None: + config = bittensor.config() + + if subtensor is None: + subtensor = bittensor.subtensor( config = config ) + + if not subtensor.is_hotkey_registered_on_subnet( + hotkey_ss58=wallet.hotkey.ss58_address, + netuid=netuid + ): + # Check if the wallet should reregister + if not config.subtensor.get('reregister'): + sys.exit(0) + + subtensor.register( + wallet = wallet, + netuid = netuid, + prompt = prompt, + TPB = config.subtensor.register.cuda.get('TPB', None), + update_interval = subtensor.register.cuda.get('update_interval', None), + num_processes = subtensor.register.get('num_processes', None), + cuda = subtensor.register.cuda.get('use_cuda', bittensor.defaults.subtensor.register.cuda.use_cuda), + dev_id = subtensor.register.cuda.get('dev_id', None), + wait_for_inclusion = wait_for_inclusion, + wait_for_finalization = wait_for_finalization, + output_in_place = subtensor.register.get('output_in_place', bittensor.defaults.subtensor.register.output_in_place), + log_verbose = subtensor.register.get('verbose', bittensor.defaults.subtensor.register.verbose), + ) + + return wallet \ No newline at end of file From c4daa9b309eb0687f23f87b01bafaa42976a4731 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 01:52:59 -0400 Subject: [PATCH 17/40] add mocks --- tests/mocks/__init__.py | 19 +++++++++ tests/mocks/keyfile_mock.py | 82 +++++++++++++++++++++++++++++++++++++ tests/mocks/wallet_mock.py | 27 ++++++++++-- 3 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 tests/mocks/__init__.py create mode 100644 tests/mocks/keyfile_mock.py diff --git a/tests/mocks/__init__.py b/tests/mocks/__init__.py new file mode 100644 index 0000000000..27846ec354 --- /dev/null +++ b/tests/mocks/__init__.py @@ -0,0 +1,19 @@ +# The MIT License (MIT) +# Copyright © 2023 Opentensor Technologies + +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the “Software”), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. + +# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +from .wallet_mock import Wallet_mock as Wallet_mock +from .keyfile_mock import Keyfile_mock as Keyfile_mock \ No newline at end of file diff --git a/tests/mocks/keyfile_mock.py b/tests/mocks/keyfile_mock.py new file mode 100644 index 0000000000..198fa0facf --- /dev/null +++ b/tests/mocks/keyfile_mock.py @@ -0,0 +1,82 @@ +# The MIT License (MIT) + +# Copyright © 2021 Yuma Rao +# Copyright © 2022 Opentensor Foundation +# Copyright © 2023 Opentensor Technologies + +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the “Software”), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. + +# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +from openwallet.keyfile_impl import serialized_keypair_to_keyfile_data, Keyfile +from openwallet import Keypair + +class MockKeyfile( Keyfile ): + """ Defines an interface to a mocked keyfile object (nothing is created on device) keypair is treated as non encrypted and the data is just the string version. + """ + def __init__( self, path: str ): + super().__init__( path ) + + self._mock_keypair = Keypair.create_from_mnemonic( mnemonic = 'arrive produce someone view end scout bargain coil slight festival excess struggle' ) + self._mock_data = serialized_keypair_to_keyfile_data( self._mock_keypair ) + + def __str__(self): + if not self.exists_on_device(): + return "Keyfile (empty, {})>".format( self.path ) + if self.is_encrypted(): + return "Keyfile (encrypted, {})>".format( self.path ) + else: + return "Keyfile (decrypted, {})>".format( self.path ) + + def __repr__(self): + return self.__str__() + + @property + def keypair( self ) -> 'Keypair': + return self._mock_keypair + + @property + def data( self ) -> bytes: + return bytes(self._mock_data) + + @property + def keyfile_data( self ) -> bytes: + return bytes( self._mock_data) + + def set_keypair ( self, keypair: 'Keypair', encrypt: bool = True, overwrite: bool = False, password:str = None): + self._mock_keypair = keypair + self._mock_data = serialized_keypair_to_keyfile_data( self._mock_keypair ) + + def get_keypair(self, password: str = None) -> 'Keypair': + return self._mock_keypair + + def make_dirs( self ): + return + + def exists_on_device( self ) -> bool: + return True + + def is_readable( self ) -> bool: + return True + + def is_writable( self ) -> bool: + return True + + def is_encrypted ( self ) -> bool: + return False + + def encrypt( self, password: str = None): + raise ValueError('Cannot encrypt a mock keyfile') + + def decrypt( self, password: str = None): + return diff --git a/tests/mocks/wallet_mock.py b/tests/mocks/wallet_mock.py index 0aa4c18a3a..cf2734478c 100644 --- a/tests/mocks/wallet_mock.py +++ b/tests/mocks/wallet_mock.py @@ -1,3 +1,22 @@ +# The MIT License (MIT) + +# Copyright © 2021 Yuma Rao +# Copyright © 2022 Opentensor Foundation +# Copyright © 2023 Opentensor Technologies + +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the “Software”), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. + +# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. import os import bittensor @@ -26,7 +45,7 @@ def __init__( print("---- MOCKED WALLET INITIALIZED- ---") @property - def hotkey_file(self) -> 'bittensor.Keyfile': + def hotkey_file(self) -> 'openwallet.Keyfile': if self._is_mock: if self._mocked_hotkey_keyfile == None: self._mocked_hotkey_keyfile = bittensor.keyfile(path='MockedHotkey', _mock = True) @@ -37,7 +56,7 @@ def hotkey_file(self) -> 'bittensor.Keyfile': return bittensor.keyfile( path = hotkey_path ) @property - def coldkey_file(self) -> 'bittensor.Keyfile': + def coldkey_file(self) -> 'openwallet.Keyfile': if self._is_mock: if self._mocked_coldkey_keyfile == None: self._mocked_coldkey_keyfile = bittensor.keyfile(path='MockedColdkey', _mock = True) @@ -48,7 +67,7 @@ def coldkey_file(self) -> 'bittensor.Keyfile': return bittensor.keyfile( path = coldkey_path ) @property - def coldkeypub_file(self) -> 'bittensor.Keyfile': + def coldkeypub_file(self) -> 'openwallet.Keyfile': if self._is_mock: if self._mocked_coldkey_keyfile == None: self._mocked_coldkey_keyfile = bittensor.keyfile(path='MockedColdkeyPub', _mock = True) @@ -56,4 +75,4 @@ def coldkeypub_file(self) -> 'bittensor.Keyfile': else: wallet_path = os.path.expanduser(os.path.join(self.path, self.name)) coldkeypub_path = os.path.join(wallet_path, "coldkeypub.txt") - return bittensor.Keyfile( path = coldkeypub_path ) \ No newline at end of file + return openwallet.Keyfile( path = coldkeypub_path ) \ No newline at end of file From 8c6cd1e270eae1b122ea43d72165215f25e437f5 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 01:53:09 -0400 Subject: [PATCH 18/40] move keyfile to package --- bittensor/_keyfile/__init__.py | 43 --- bittensor/_keyfile/keyfile_impl.py | 557 ----------------------------- 2 files changed, 600 deletions(-) delete mode 100644 bittensor/_keyfile/__init__.py delete mode 100644 bittensor/_keyfile/keyfile_impl.py diff --git a/bittensor/_keyfile/__init__.py b/bittensor/_keyfile/__init__.py deleted file mode 100644 index 9c34fcda5b..0000000000 --- a/bittensor/_keyfile/__init__.py +++ /dev/null @@ -1,43 +0,0 @@ -# The MIT License (MIT) -# Copyright © 2021 Yuma Rao - -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the “Software”), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. - -# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -import bittensor -from . import keyfile_impl - -class keyfile (object): - """ Factory for a bittensor on device keypair - """ - def __new__( - cls, - path: str = None, - _mock: bool = False - ) -> 'bittensor.Keyfile': - r""" Initialize a bittensor on device keypair interface. - - Args: - path (required=False, default: ~/.bittensor/wallets/default/coldkey ): - Path where this keypair is stored. - """ - path = '~/.bittensor/wallets/default/coldkey' if path == None else path - if _mock: - return keyfile_impl.MockKeyfile( path = path ) - else: - return keyfile_impl.Keyfile( path = path ) - - @classmethod - def mock(cls): - return keyfile(_mock=True) diff --git a/bittensor/_keyfile/keyfile_impl.py b/bittensor/_keyfile/keyfile_impl.py deleted file mode 100644 index bfb6a6d95e..0000000000 --- a/bittensor/_keyfile/keyfile_impl.py +++ /dev/null @@ -1,557 +0,0 @@ -# The MIT License (MIT) -# Copyright © 2021 Yuma Rao - -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the “Software”), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. - -# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -import os -import base64 -import json -import stat -import getpass -import bittensor -from typing import Optional -from pathlib import Path - -from ansible_vault import Vault -from ansible.parsing.vault import AnsibleVaultError -from cryptography.exceptions import InvalidSignature, InvalidKey -from cryptography.fernet import Fernet, InvalidToken -from cryptography.hazmat.primitives import hashes -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC -from password_strength import PasswordPolicy -from substrateinterface.utils.ss58 import ss58_encode -from termcolor import colored - -class KeyFileError(Exception): - """ Error thrown when the keyfile is corrupt, non-writable, nno-readable or the password used to decrypt is invalid. - """ - -def serialized_keypair_to_keyfile_data( keypair: 'bittensor.Keypair' ): - """ Serializes keypair object into keyfile data. - Args: - password ( str, required ): - password to verify. - Returns: - valid ( bool ): - True if the password meets validity requirements. - """ - json_data = { - 'accountId': "0x" + keypair.public_key.hex() if keypair.public_key != None else None, - 'publicKey': "0x" + keypair.public_key.hex() if keypair.public_key != None else None, - 'secretPhrase': keypair.mnemonic if keypair.mnemonic != None else None, - 'secretSeed': "0x" + \ - # If bytes -> str - ( keypair.seed_hex if isinstance(keypair.seed_hex, str) else keypair.seed_hex.hex() ) - # If None -> None - if keypair.seed_hex != None else None, - 'ss58Address': keypair.ss58_address if keypair.ss58_address != None else None - } - data = json.dumps( json_data ).encode() - return data - -def deserialize_keypair_from_keyfile_data( keyfile_data:bytes ) -> 'bittensor.Keypair': - """ Deserializes Keypair object from passed keyfile data. - Args: - keyfile_data ( bytest, required ): - Keyfile data as bytes to be loaded. - Returns: - keypair (bittensor.Keypair): - Keypair loaded from bytes. - Raises: - KeyFileError: - Raised if the passed bytest cannot construct a keypair object. - """ - # Decode from json. - keyfile_data = keyfile_data.decode() - try: - keyfile_dict = dict(json.loads( keyfile_data )) - except: - string_value = str(keyfile_data) - if string_value[:2] == "0x": - string_value = ss58_encode( string_value ) - keyfile_dict = { - 'accountId': None, - 'publicKey': None, - 'secretPhrase': None, - 'secretSeed': None, - 'ss58Address': string_value - } - else: - raise KeyFileError('Keypair could not be created from keyfile data: {}'.format( string_value )) - - if "secretSeed" in keyfile_dict and keyfile_dict['secretSeed'] != None: - return bittensor.Keypair.create_from_seed(keyfile_dict['secretSeed']) - - if "secretPhrase" in keyfile_dict and keyfile_dict['secretPhrase'] != None: - return bittensor.Keypair.create_from_mnemonic(mnemonic=keyfile_dict['secretPhrase']) - - if "ss58Address" in keyfile_dict and keyfile_dict['ss58Address'] != None: - return bittensor.Keypair( ss58_address = keyfile_dict['ss58Address'] ) - - else: - raise KeyFileError('Keypair could not be created from keyfile data: {}'.format( keyfile_dict )) - -def validate_password( password:str ) -> bool: - """ Validates the password again a password policy. - Args: - password ( str, required ): - password to verify. - Returns: - valid ( bool ): - True if the password meets validity requirements. - """ - policy = PasswordPolicy.from_names( - strength=0.20, - entropybits=10, - length=6, - ) - if not password: - return False - tested_pass = policy.password(password) - result = tested_pass.test() - if len(result) > 0: - print(colored('Password not strong enough. Try increasing the length of the password or the password complexity')) - return False - password_verification = getpass.getpass("Retype your password: ") - if password != password_verification: - print("Passwords do not match") - return False - return True - -def ask_password_to_encrypt() -> str: - """ Password from user prompt. - Returns: - password (str): - Valid password from user prompt. - """ - valid = False - while not valid: - password = getpass.getpass("Specify password for key encryption: ") - valid = validate_password(password) - return password - -def keyfile_data_is_encrypted_ansible( keyfile_data:bytes ) -> bool: - """ Returns true if the keyfile data is ansible encrypted. - Args: - keyfile_data ( bytes, required ): - Bytes to validate - Returns: - is_ansible (bool): - True if data is ansible encrypted. - """ - return keyfile_data[:14] == b'$ANSIBLE_VAULT' - -def keyfile_data_is_encrypted_legacy( keyfile_data:bytes ) -> bool: - """ Returns true if the keyfile data is legacy encrypted. - Args: - keyfile_data ( bytes, required ): - Bytes to validate - Returns: - is_legacy (bool): - True if data is legacy encrypted. - """ - return keyfile_data[:6] == b"gAAAAA" - -def keyfile_data_is_encrypted( keyfile_data:bytes ) -> bool: - """ Returns true if the keyfile data is encrypted. - Args: - keyfile_data ( bytes, required ): - Bytes to validate - Returns: - is_encrypted (bool): - True if data is encrypted. - """ - return keyfile_data_is_encrypted_ansible( keyfile_data ) or keyfile_data_is_encrypted_legacy( keyfile_data ) - -def encrypt_keyfile_data ( keyfile_data:bytes, password: str = None ) -> bytes: - """ Encrypts passed keyfile data using ansible vault. - Args: - keyfile_data ( bytes, required ): - Bytes to validate - password ( bool, optional ): - It set, uses this password to encrypt data. - Returns: - encrytped_data (bytes): - Ansible encrypted data. - """ - password = ask_password_to_encrypt() if password == None else password - console = bittensor.__console__; - with console.status(":locked_with_key: Encrypting key..."): - vault = Vault( password ) - return vault.vault.encrypt ( keyfile_data ) - - -def get_coldkey_password_from_environment(coldkey_name: str) -> Optional[str]: - - for env_var in os.environ: - if ( - env_var.upper().startswith("BT_COLD_PW_") - and env_var.upper().endswith(coldkey_name.upper()) - ): - return os.getenv(env_var) - - return None - - -def decrypt_keyfile_data(keyfile_data: bytes, password: str = None, coldkey_name: Optional[str] = None) -> bytes: - """ Decrypts passed keyfile data using ansible vault. - Args: - keyfile_data ( bytes, required ): - Bytes to validate - password ( bool, optional ): - It set, uses this password to decrypt data. - Returns: - decrypted_data (bytes): - Decrypted data. - Raises: - KeyFileError: - Raised if the file is corrupted or if the password is incorrect. - """ - if coldkey_name is not None and password is None: - password = get_coldkey_password_from_environment(coldkey_name) - - try: - password = getpass.getpass("Enter password to unlock key: ") if password is None else password - console = bittensor.__console__; - with console.status(":key: Decrypting key..."): - # Ansible decrypt. - if keyfile_data_is_encrypted_ansible( keyfile_data ): - vault = Vault( password ) - try: - decrypted_keyfile_data = vault.load( keyfile_data ) - except AnsibleVaultError: - raise KeyFileError('Invalid password') - # Legacy decrypt. - elif keyfile_data_is_encrypted_legacy( keyfile_data ): - __SALT = b"Iguesscyborgslikemyselfhaveatendencytobeparanoidaboutourorigins" - kdf = PBKDF2HMAC(algorithm=hashes.SHA256(), salt=__SALT, length=32, iterations=10000000, backend=default_backend()) - key = base64.urlsafe_b64encode(kdf.derive(password.encode())) - cipher_suite = Fernet(key) - decrypted_keyfile_data = cipher_suite.decrypt( keyfile_data ) - # Unknown. - else: - raise KeyFileError( "Keyfile data: {} is corrupt".format( keyfile_data )) - - except (InvalidSignature, InvalidKey, InvalidToken): - raise KeyFileError('Invalid password') - - if not isinstance(decrypted_keyfile_data, bytes): - decrypted_keyfile_data = json.dumps( decrypted_keyfile_data ).encode() - return decrypted_keyfile_data - -class Keyfile( object ): - """ Defines an interface for a subtrate interface keypair stored on device. - """ - def __init__( self, path: str ): - self.path = os.path.expanduser(path) - self.name = Path(self.path).parent.stem - - def __str__(self): - if not self.exists_on_device(): - return "Keyfile (empty, {})>".format( self.path ) - if self.is_encrypted(): - return "Keyfile (encrypted, {})>".format( self.path ) - else: - return "Keyfile (decrypted, {})>".format( self.path ) - - def __repr__(self): - return self.__str__() - - @property - def keypair( self ) -> 'bittensor.Keypair': - """ Returns the keypair from path, decrypts data if the file is encrypted. - Args: - password ( str, optional ): - Optional password used to decrypt file. If None, asks for user input. - Returns: - keypair (bittensor.Keypair): - Keypair stored under path. - Raises: - KeyFileError: - Raised if the file does not exists, is not readable, writable - corrupted, or if the password is incorrect. - """ - return self.get_keypair() - - @property - def data( self ) -> bytes: - """ Returns keyfile data under path. - Returns: - keyfile_data (bytes): - Keyfile data stored under path. - Raises: - KeyFileError: - Raised if the file does not exists, is not readable, or writable. - """ - return self._read_keyfile_data_from_file() - - @property - def keyfile_data( self ) -> bytes: - """ Returns keyfile data under path. - Returns: - keyfile_data (bytes): - Keyfile data stored under path. - Raises: - KeyFileError: - Raised if the file does not exists, is not readable, or writable. - """ - return self._read_keyfile_data_from_file() - - def set_keypair ( self, keypair: 'bittensor.Keypair', encrypt: bool = True, overwrite: bool = False, password:str = None): - """ Writes the keypair to the file and optional encrypts data. - Args: - keypair (bittensor.Keypair): - Keypair to store under path. - encrypt ( bool, optional, default = True ): - If True, encrypts file under path. - overwrite ( bool, optional, default = True ): - If True, forces overwrite of current file. - password ( str, optional ): - Optional password used to encrypt file. If None, asks for user input. - Raises: - KeyFileError: - Raised if the file does not exists, is not readable, or writable. - """ - self.make_dirs() - keyfile_data = serialized_keypair_to_keyfile_data( keypair ) - if encrypt: - keyfile_data = encrypt_keyfile_data( keyfile_data, password ) - self._write_keyfile_data_to_file( keyfile_data, overwrite = overwrite ) - - def get_keypair(self, password: str = None) -> 'bittensor.Keypair': - """ Returns the keypair from path, decrypts data if the file is encrypted. - Args: - password ( str, optional ): - Optional password used to decrypt file. If None, asks for user input. - Returns: - keypair (bittensor.Keypair): - Keypair stored under path. - Raises: - KeyFileError: - Raised if the file does not exists, is not readable, writable - corrupted, or if the password is incorrect. - """ - keyfile_data = self._read_keyfile_data_from_file() - if keyfile_data_is_encrypted( keyfile_data ): - keyfile_data = decrypt_keyfile_data(keyfile_data, password, coldkey_name=self.name) - return deserialize_keypair_from_keyfile_data( keyfile_data ) - - def make_dirs( self ): - """ Makes directories for path. - """ - directory = os.path.dirname( self.path ) - if not os.path.exists( directory ): - os.makedirs( directory ) - - def exists_on_device( self ) -> bool: - """ Returns true if the file exists on the device. - Returns: - on_device (bool): - True if the file is on device. - """ - if not os.path.isfile( self.path ): - return False - return True - - def is_readable( self ) -> bool: - """ Returns true if the file under path is readable. - Returns: - readable (bool): - True if the file is readable. - """ - if not self.exists_on_device(): - return False - if not os.access( self.path , os.R_OK ): - return False - return True - - def is_writable( self ) -> bool: - """ Returns true if the file under path is writable. - Returns: - writable (bool): - True if the file is writable. - """ - if os.access(self.path, os.W_OK): - return True - return False - - def is_encrypted ( self ) -> bool: - """ Returns true if the file under path is encrypted. - Returns: - encrypted (bool): - True if the file is encrypted. - """ - if not self.exists_on_device(): - return False - if not self.is_readable(): - return False - return keyfile_data_is_encrypted( self._read_keyfile_data_from_file() ) - - def _may_overwrite ( self ) -> bool: - choice = input("File {} already exists. Overwrite ? (y/N) ".format( self.path )) - return choice == 'y' - - def encrypt( self, password: str = None): - """ Encrypts file under path. - Args: - password: (str, optional): - Optional password for encryption. Otherwise asks for user input. - Raises: - KeyFileError: - Raised if the file does not exists, is not readable, writable. - """ - if not self.exists_on_device(): - raise KeyFileError( "Keyfile at: {} is not a file".format( self.path )) - if not self.is_readable(): - raise KeyFileError( "Keyfile at: {} is not readable".format( self.path )) - if not self.is_writable(): - raise KeyFileError( "Keyfile at: {} is not writeable".format( self.path ) ) - keyfile_data = self._read_keyfile_data_from_file() - if not keyfile_data_is_encrypted( keyfile_data ): - as_keypair = deserialize_keypair_from_keyfile_data( keyfile_data ) - keyfile_data = serialized_keypair_to_keyfile_data( as_keypair ) - keyfile_data = encrypt_keyfile_data( keyfile_data, password ) - self._write_keyfile_data_to_file( keyfile_data, overwrite = True ) - - def decrypt( self, password: str = None): - """ Decrypts file under path. - Args: - password: (str, optional): - Optional password for decryption. Otherwise asks for user input. - Raises: - KeyFileError: - Raised if the file does not exists, is not readable, writable - corrupted, or if the password is incorrect. - """ - if not self.exists_on_device(): - raise KeyFileError( "Keyfile at: {} is not a file".format( self.path )) - if not self.is_readable(): - raise KeyFileError( "Keyfile at: {} is not readable".format( self.path )) - if not self.is_writable(): - raise KeyFileError( "No write access for {}".format( self.path ) ) - keyfile_data = self._read_keyfile_data_from_file() - if keyfile_data_is_encrypted( keyfile_data ): - keyfile_data = decrypt_keyfile_data(keyfile_data, password, coldkey_name=self.name) - as_keypair = deserialize_keypair_from_keyfile_data( keyfile_data ) - keyfile_data = serialized_keypair_to_keyfile_data( as_keypair ) - self._write_keyfile_data_to_file( keyfile_data, overwrite = True ) - - def _read_keyfile_data_from_file ( self ) -> bytes: - """ Reads keyfile data from path. - Returns: - keyfile_data: (bytes, required): - Keyfile data sotred under path. - Raises: - KeyFileError: - Raised if the file does not exists or is not readable. - """ - if not self.exists_on_device(): - raise KeyFileError( "Keyfile at: {} is not a file".format( self.path )) - if not self.is_readable(): - raise KeyFileError( "Keyfile at: {} is not readable".format( self.path )) - with open( self.path , 'rb') as file: - data = file.read() - return data - - def _write_keyfile_data_to_file ( self, keyfile_data:bytes, overwrite: bool = False ): - """ Writes the keyfile data to path, if overwrite is true, forces operation without asking. - Args: - keyfile_data: (bytes, required): - Byte data to store under path. - overwrite (bool, optional): - If True, overwrites data without asking for overwrite permissions from the user. - Raises: - KeyFileError: - Raised if the file is not writable or the user returns No to overwrite prompt. - """ - # Check overwrite. - if self.exists_on_device() and not overwrite: - if not self._may_overwrite(): - raise KeyFileError( "Keyfile at: {} is not writeable".format( self.path ) ) - with open(self.path, "wb") as keyfile: - keyfile.write( keyfile_data ) - # Set file permissions. - os.chmod(self.path, stat.S_IRUSR | stat.S_IWUSR) - - -class MockKeyfile( object ): - """ Defines an interface to a mocked keyfile object (nothing is created on device) keypair is treated as non encrypted and the data is just the string version. - """ - def __init__( self, path: str ): - self.path = os.path.expanduser(path) - self._mock_keypair = bittensor.Keypair.create_from_mnemonic( mnemonic = 'arrive produce someone view end scout bargain coil slight festival excess struggle' ) - self._mock_data = serialized_keypair_to_keyfile_data( self._mock_keypair ) - - def __str__(self): - if not self.exists_on_device(): - return "Keyfile (empty, {})>".format( self.path ) - if self.is_encrypted(): - return "Keyfile (encrypted, {})>".format( self.path ) - else: - return "Keyfile (decrypted, {})>".format( self.path ) - - def __repr__(self): - return self.__str__() - - @property - def keypair( self ) -> 'bittensor.Keypair': - return self._mock_keypair - - @property - def data( self ) -> bytes: - return bytes(self._mock_data) - - @property - def keyfile_data( self ) -> bytes: - return bytes( self._mock_data) - - def set_keypair ( self, keypair: 'bittensor.Keypair', encrypt: bool = True, overwrite: bool = False, password:str = None): - self._mock_keypair = keypair - self._mock_data = serialized_keypair_to_keyfile_data( self._mock_keypair ) - - def get_keypair(self, password: str = None) -> 'bittensor.Keypair': - return self._mock_keypair - - def make_dirs( self ): - return - - def exists_on_device( self ) -> bool: - return True - - def is_readable( self ) -> bool: - return True - - def is_writable( self ) -> bool: - return True - - def is_encrypted ( self ) -> bool: - return False - - def encrypt( self, password: str = None): - raise ValueError('Cannot encrypt a mock keyfile') - - def decrypt( self, password: str = None): - return - - - - - - - - - - From 1664def826c95e7376fb91a6d78a8d20e926944a Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 01:53:31 -0400 Subject: [PATCH 19/40] move test to utils tests --- .../unit_tests/bittensor_tests/test_wallet.py | 158 --------------- .../bittensor_tests/utils/test_utils.py | 187 +++++++++++++++++- 2 files changed, 186 insertions(+), 159 deletions(-) delete mode 100644 tests/unit_tests/bittensor_tests/test_wallet.py diff --git a/tests/unit_tests/bittensor_tests/test_wallet.py b/tests/unit_tests/bittensor_tests/test_wallet.py deleted file mode 100644 index b721122dd8..0000000000 --- a/tests/unit_tests/bittensor_tests/test_wallet.py +++ /dev/null @@ -1,158 +0,0 @@ -# The MIT License (MIT) -# Copyright © 2022 Yuma Rao - -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the “Software”), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. - -# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -import unittest -from unittest.mock import patch, MagicMock -import pytest -import bittensor - - -class TestWalletReregister(unittest.TestCase): - def test_wallet_reregister_use_cuda_flag_none(self): - config = bittensor.Config() - config.wallet = bittensor.Config() - config.wallet.reregister = True - - config.subtensor = bittensor.Config() - config.subtensor.register = bittensor.Config() - config.subtensor.register.cuda = bittensor.Config() - config.subtensor.register.cuda.use_cuda = None # don't set the argument, but do specify the flag - # No need to specify the other config options as they are default to None - - mock_wallet = bittensor.wallet.mock() - mock_wallet.is_registered = MagicMock(return_value=False) - mock_wallet.config = config - - class MockException(Exception): - pass - - def exit_early(*args, **kwargs): - raise MockException('exit_early') - - with patch('bittensor.Subtensor.register', side_effect=exit_early) as mock_register: - # Should be able to set without argument - with pytest.raises(MockException): - mock_wallet.reregister( netuid = -1 ) - - call_args = mock_register.call_args - _, kwargs = call_args - - mock_register.assert_called_once() - self.assertEqual(kwargs['cuda'], None) # should be None when no argument, but flag set - - def test_wallet_reregister_use_cuda_flag_true(self): - config = bittensor.Config() - config.wallet = bittensor.Config() - config.wallet.reregister = True - - config.subtensor = bittensor.Config() - config.subtensor.register = bittensor.Config() - config.subtensor.register.cuda = bittensor.Config() - config.subtensor.register.cuda.use_cuda = True - config.subtensor.register.cuda.dev_id = 0 - # No need to specify the other config options as they are default to None - - mock_wallet = bittensor.wallet.mock() - mock_wallet.is_registered = MagicMock(return_value=False) - mock_wallet.config = config - - class MockException(Exception): - pass - - def exit_early(*args, **kwargs): - raise MockException('exit_early') - - with patch('bittensor.Subtensor.register', side_effect=exit_early) as mock_register: - # Should be able to set without argument - with pytest.raises(MockException): - mock_wallet.reregister( netuid = -1 ) - - call_args = mock_register.call_args - _, kwargs = call_args - - mock_register.assert_called_once() - self.assertEqual(kwargs['cuda'], True) # should be default when no argument - - def test_wallet_reregister_use_cuda_flag_false(self): - config = bittensor.Config() - config.wallet = bittensor.Config() - config.wallet.reregister = True - - config.subtensor = bittensor.Config() - config.subtensor.register = bittensor.Config() - config.subtensor.register.cuda = bittensor.Config() - config.subtensor.register.cuda.use_cuda = False - config.subtensor.register.cuda.dev_id = 0 - # No need to specify the other config options as they are default to None - - mock_wallet = bittensor.wallet.mock() - mock_wallet.is_registered = MagicMock(return_value=False) - mock_wallet.config = config - - class MockException(Exception): - pass - - def exit_early(*args, **kwargs): - raise MockException('exit_early') - - with patch('bittensor.Subtensor.register', side_effect=exit_early) as mock_register: - # Should be able to set without argument - with pytest.raises(MockException): - mock_wallet.reregister( netuid = -1 ) - - call_args = mock_register.call_args - _, kwargs = call_args - - mock_register.assert_called_once() - self.assertEqual(kwargs['cuda'], False) # should be default when no argument - - def test_wallet_reregister_use_cuda_flag_not_specified_false(self): - config = bittensor.Config() - config.wallet = bittensor.Config() - config.wallet.reregister = True - - config.subtensor = bittensor.Config() - config.subtensor.register = bittensor.Config() - config.subtensor.register.cuda = bittensor.Config() - #config.subtensor.register.cuda.use_cuda # don't specify the flag - config.subtensor.register.cuda.dev_id = 0 - # No need to specify the other config options as they are default to None - - mock_wallet = bittensor.wallet.mock() - mock_wallet.is_registered = MagicMock(return_value=False) - mock_wallet.config = config - - class MockException(Exception): - pass - - def exit_early(*args, **kwargs): - raise MockException('exit_early') - - with patch('bittensor.Subtensor.register', side_effect=exit_early) as mock_register: - # Should be able to set without argument - with pytest.raises(MockException): - mock_wallet.reregister( netuid = -1 ) - - call_args = mock_register.call_args - _, kwargs = call_args - - mock_register.assert_called_once() - self.assertEqual(kwargs['cuda'], False) # should be False when no flag was set - - -if __name__ == '__main__': - unittest.main() \ No newline at end of file diff --git a/tests/unit_tests/bittensor_tests/utils/test_utils.py b/tests/unit_tests/bittensor_tests/utils/test_utils.py index 7291b8d36e..956e11758f 100644 --- a/tests/unit_tests/bittensor_tests/utils/test_utils.py +++ b/tests/unit_tests/bittensor_tests/utils/test_utils.py @@ -24,6 +24,9 @@ import bittensor from bittensor.utils.registration import _CUDASolver, _SolverBase +from bittensor._subtensor.subtensor_mock import MockSubtensor + +from tests.mocks.wallet_mock import Wallet_mock @fixture(scope="function") @@ -67,7 +70,7 @@ def select_port(): return port def generate_wallet(coldkey : 'Keypair' = None, hotkey: 'Keypair' = None): - wallet = bittensor.wallet(_mock=True) + wallet = Wallet_mock( _mock = True, name = 'mock', path = '/tmp/', hotkey = 'mock' ) if not coldkey: coldkey = Keypair.create_from_mnemonic(Keypair.generate_mnemonic()) @@ -651,5 +654,187 @@ def test_get_explorer_url_for_network_by_network_and_block_hash(self, network: s self.assertEqual(bittensor.utils.get_explorer_url_for_network(network, block_hash, self.network_map), expected) +class TestWalletReregister(unittest.TestCase): + _mock_subtensor: MockSubtensor + + def setUp(self): + self.subtensor = bittensor.subtensor( network = 'mock' ) # own instance per test + + @classmethod + def setUpClass(cls) -> None: + # Keeps the same mock network for all tests. This stops the network from being re-setup for each test. + cls._mock_subtensor = bittensor.subtensor( network = 'mock' ) + + cls._do_setup_subnet() + + @classmethod + def _do_setup_subnet(cls): + # reset the mock subtensor + cls._mock_subtensor.reset() + # Setup the mock subnet 3 + cls._mock_subtensor.create_subnet( + netuid = 3 + ) + + def test_wallet_reregister_reregister_false(self): + mock_wallet = generate_wallet() + + class MockException(Exception): + pass + + with patch('bittensor.Subtensor.register', side_effect=MockException) as mock_register: + with pytest.raises(SystemExit): # should exit because it's not registered + bittensor.utils.reregister( + wallet = mock_wallet, + subtensor = self._mock_subtensor, + netuid = 3, + reregister = False, + ) + + mock_register.assert_not_called() # should not call register + + def test_wallet_reregister_reregister_false_and_registered_already(self): + mock_wallet = generate_wallet() + + class MockException(Exception): + pass + + self._mock_subtensor.force_register_neuron( + netuid = 3, + hotkey = mock_wallet.hotkey.ss58_address, + coldkey = mock_wallet.coldkeypub.ss58_address, + ) + self.assertTrue(self._mock_subtensor.is_hotkey_registered_on_subnet( + netuid = 3, + hotkey_ss58 = mock_wallet.hotkey.ss58_address, + )) + + with patch('bittensor.Subtensor.register', side_effect=MockException) as mock_register: + bittensor.utils.reregister( + wallet = mock_wallet, + subtensor = self._mock_subtensor, + netuid = 3, + reregister = False, + ) # Should not exit because it's registered + + mock_register.assert_not_called() # should not call register + + def test_wallet_reregister_reregister_true_and_registered_already(self): + mock_wallet = generate_wallet() + + class MockException(Exception): + pass + + self._mock_subtensor.force_register_neuron( + netuid = 3, + hotkey = mock_wallet.hotkey.ss58_address, + coldkey = mock_wallet.coldkeypub.ss58_address, + ) + self.assertTrue(self._mock_subtensor.is_hotkey_registered_on_subnet( + netuid = 3, + hotkey_ss58 = mock_wallet.hotkey.ss58_address, + )) + + with patch('bittensor.Subtensor.register', side_effect=MockException) as mock_register: + bittensor.utils.reregister( + wallet = mock_wallet, + subtensor = self._mock_subtensor, + netuid = 3, + reregister = True, + ) # Should not exit because it's registered + + mock_register.assert_not_called() # should not call register + + + def test_wallet_reregister_no_params(self): + mock_wallet = generate_wallet() + + class MockException(Exception): + pass + + with patch('bittensor.Subtensor.register', side_effect=MockException) as mock_register: + # Should be able to set without argument + with pytest.raises(MockException): + bittensor.utils.reregister( + wallet = mock_wallet, + subtensor = self._mock_subtensor, + netuid = 3, + reregister = True, + # didn't pass any register params + ) + + mock_register.assert_called_once() # should call register once + + def test_wallet_reregister_use_cuda_flag_true(self): + mock_wallet = generate_wallet() + + class MockException(Exception): + pass + + with patch('bittensor.Subtensor.register', side_effect=MockException) as mock_register: + # Should be able to set without argument + with pytest.raises(MockException): + bittensor.utils.reregister( + wallet = mock_wallet, + subtensor = self._mock_subtensor, + netuid = 3, + dev_id = 0, + use_cuda = True, + reregister = True, + ) + + call_args = mock_register.call_args + _, kwargs = call_args + + mock_register.assert_called_once() + self.assertEqual(kwargs['cuda'], True) + + def test_wallet_reregister_use_cuda_flag_false(self): + mock_wallet = generate_wallet() + + class MockException(Exception): + pass + + with patch('bittensor.Subtensor.register', side_effect=MockException) as mock_register: + # Should be able to set without argument + with pytest.raises(MockException): + bittensor.utils.reregister( + wallet = mock_wallet, + subtensor = self._mock_subtensor, + netuid = 3, + dev_id = 0, + use_cuda = False, + reregister = True, + ) + + call_args = mock_register.call_args + _, kwargs = call_args + + mock_register.assert_called_once() + self.assertEqual(kwargs['cuda'], False) + + def test_wallet_reregister_use_cuda_flag_not_specified_false(self): + mock_wallet = generate_wallet() + + class MockException(Exception): + pass + + with patch('bittensor.Subtensor.register', side_effect=MockException) as mock_register: + # Should be able to set without argument + with pytest.raises(MockException): + bittensor.utils.reregister( + wallet = mock_wallet, + subtensor = self._mock_subtensor, + netuid = 3, + dev_id = 0, + ) + + call_args = mock_register.call_args + _, kwargs = call_args + + mock_register.assert_called_once() + self.assertEqual(kwargs['cuda'], False) # should be False by default + + if __name__ == "__main__": unittest.main() From ac751bf5e42fe932fd64d173fb676d8e969db73b Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 01:55:25 -0400 Subject: [PATCH 20/40] use different naming --- tests/integration_tests/test_cli.py | 2 +- tests/mocks/__init__.py | 4 ++-- tests/mocks/wallet_mock.py | 2 +- tests/unit_tests/bittensor_tests/utils/test_utils.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/integration_tests/test_cli.py b/tests/integration_tests/test_cli.py index e2bf0dda39..b7626f0a66 100644 --- a/tests/integration_tests/test_cli.py +++ b/tests/integration_tests/test_cli.py @@ -2237,7 +2237,7 @@ def test_delegate(self): ) with patch( - "bittensor._wallet.wallet_mock.Wallet_mock", return_value=mock_wallet + "tests.mocks.wallet_mock.MockWallet", return_value=mock_wallet ): # Mock wallet creation. SHOULD NOT BE REGISTERED cli = bittensor.cli( args=[ diff --git a/tests/mocks/__init__.py b/tests/mocks/__init__.py index 27846ec354..a5edffa7bd 100644 --- a/tests/mocks/__init__.py +++ b/tests/mocks/__init__.py @@ -15,5 +15,5 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. -from .wallet_mock import Wallet_mock as Wallet_mock -from .keyfile_mock import Keyfile_mock as Keyfile_mock \ No newline at end of file +from .wallet_mock import MockWallet as MockWallet +from .keyfile_mock import MockKeyfile as MockKeyfile \ No newline at end of file diff --git a/tests/mocks/wallet_mock.py b/tests/mocks/wallet_mock.py index cf2734478c..6d9a8f5031 100644 --- a/tests/mocks/wallet_mock.py +++ b/tests/mocks/wallet_mock.py @@ -22,7 +22,7 @@ import bittensor import openwallet -class Wallet_mock(openwallet.Wallet): +class MockWallet(openwallet.Wallet): """ Mocked Version of the bittensor wallet class, meant to be used for testing """ diff --git a/tests/unit_tests/bittensor_tests/utils/test_utils.py b/tests/unit_tests/bittensor_tests/utils/test_utils.py index 956e11758f..c000e4d14f 100644 --- a/tests/unit_tests/bittensor_tests/utils/test_utils.py +++ b/tests/unit_tests/bittensor_tests/utils/test_utils.py @@ -26,7 +26,7 @@ from bittensor.utils.registration import _CUDASolver, _SolverBase from bittensor._subtensor.subtensor_mock import MockSubtensor -from tests.mocks.wallet_mock import Wallet_mock +from tests.mocks.wallet_mock import MockWallet @fixture(scope="function") @@ -70,7 +70,7 @@ def select_port(): return port def generate_wallet(coldkey : 'Keypair' = None, hotkey: 'Keypair' = None): - wallet = Wallet_mock( _mock = True, name = 'mock', path = '/tmp/', hotkey = 'mock' ) + wallet = MockWallet( _mock = True, name = 'mock', path = '/tmp/', hotkey = 'mock' ) if not coldkey: coldkey = Keypair.create_from_mnemonic(Keypair.generate_mnemonic()) From 485e4d56b100677227277a5d7d0e7d646cf1762d Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 01:58:42 -0400 Subject: [PATCH 21/40] fix import --- tests/mocks/wallet_mock.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/mocks/wallet_mock.py b/tests/mocks/wallet_mock.py index 6d9a8f5031..0715abadc9 100644 --- a/tests/mocks/wallet_mock.py +++ b/tests/mocks/wallet_mock.py @@ -22,6 +22,8 @@ import bittensor import openwallet +from .keyfile_mock import MockKeyfile + class MockWallet(openwallet.Wallet): """ Mocked Version of the bittensor wallet class, meant to be used for testing @@ -48,7 +50,7 @@ def __init__( def hotkey_file(self) -> 'openwallet.Keyfile': if self._is_mock: if self._mocked_hotkey_keyfile == None: - self._mocked_hotkey_keyfile = bittensor.keyfile(path='MockedHotkey', _mock = True) + self._mocked_hotkey_keyfile = MockKeyfile(path='MockedHotkey') return self._mocked_hotkey_keyfile else: wallet_path = os.path.expanduser(os.path.join(self.path, self.name)) @@ -59,7 +61,7 @@ def hotkey_file(self) -> 'openwallet.Keyfile': def coldkey_file(self) -> 'openwallet.Keyfile': if self._is_mock: if self._mocked_coldkey_keyfile == None: - self._mocked_coldkey_keyfile = bittensor.keyfile(path='MockedColdkey', _mock = True) + self._mocked_coldkey_keyfile = MockKeyfile(path='MockedColdkey') return self._mocked_coldkey_keyfile else: wallet_path = os.path.expanduser(os.path.join(self.path, self.name)) @@ -70,7 +72,7 @@ def coldkey_file(self) -> 'openwallet.Keyfile': def coldkeypub_file(self) -> 'openwallet.Keyfile': if self._is_mock: if self._mocked_coldkey_keyfile == None: - self._mocked_coldkey_keyfile = bittensor.keyfile(path='MockedColdkeyPub', _mock = True) + self._mocked_coldkey_keyfile = MockKeyfile(path='MockedColdkeyPub') return self._mocked_coldkey_keyfile else: wallet_path = os.path.expanduser(os.path.join(self.path, self.name)) From 001538c559403123200a235449857abd49609986 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 01:58:51 -0400 Subject: [PATCH 22/40] change function name and use kwargs --- bittensor/utils/registration.py | 49 ++++++++++----------------------- 1 file changed, 14 insertions(+), 35 deletions(-) diff --git a/bittensor/utils/registration.py b/bittensor/utils/registration.py index b5378f7b7a..967aea36a2 100644 --- a/bittensor/utils/registration.py +++ b/bittensor/utils/registration.py @@ -872,39 +872,32 @@ def create_pow( return solution -def reregister_wallet( +def reregister( netuid: int, wallet: 'bittensor.Wallet', - subtensor: Optional['bittensor.Subtensor'] = None, - config: Optional['bittensor.Config'] = None, - wait_for_inclusion: bool = False, - wait_for_finalization: bool = True, - prompt: bool = False + subtensor: 'bittensor.Subtensor', + reregister: bool = False, + prompt: bool = False, + **registration_args: Any ) -> Optional['bittensor.Wallet']: """ Re-register this a Wallet on the chain, or exits. Exits if the wallet is not registered on the chain AND - the config.subtensor.reregister flag is set to False. + reregister is set to False. Args: netuid (int): The network uid of the subnet to register on. wallet( 'bittensor.Wallet' ): Bittensor Wallet to re-register - subtensor (Optional['bittensor.Subtensor']): - Bittensor subtensor to use for registration. - config (Optional['bittensor.Config']): - Bittensor config to use for registration. - wait_for_inclusion (bool): - if set, waits for the extrinsic to enter a block before returning true, - or returns false if the extrinsic fails to enter the block within the timeout. - wait_for_finalization (bool): - if set, waits for the extrinsic to be finalized on the chain before returning true, - or returns false if the extrinsic fails to be finalized within the timeout. + reregister (bool, default=False): + If true, re-registers the wallet on the chain. + Exits if False and the wallet is not registered on the chain. prompt (bool): If true, the call waits for confirmation from the user before proceeding. - + **registration_args (Any): + The registration arguments to pass to the subtensor register function. Return: wallet (bittensor.Wallet): - This wallet. + The wallet Raises: SytemExit(0): @@ -913,33 +906,19 @@ def reregister_wallet( """ wallet.hotkey - if config is None: - config = bittensor.config() - - if subtensor is None: - subtensor = bittensor.subtensor( config = config ) - if not subtensor.is_hotkey_registered_on_subnet( hotkey_ss58=wallet.hotkey.ss58_address, netuid=netuid ): # Check if the wallet should reregister - if not config.subtensor.get('reregister'): + if not reregister: sys.exit(0) subtensor.register( wallet = wallet, netuid = netuid, prompt = prompt, - TPB = config.subtensor.register.cuda.get('TPB', None), - update_interval = subtensor.register.cuda.get('update_interval', None), - num_processes = subtensor.register.get('num_processes', None), - cuda = subtensor.register.cuda.get('use_cuda', bittensor.defaults.subtensor.register.cuda.use_cuda), - dev_id = subtensor.register.cuda.get('dev_id', None), - wait_for_inclusion = wait_for_inclusion, - wait_for_finalization = wait_for_finalization, - output_in_place = subtensor.register.get('output_in_place', bittensor.defaults.subtensor.register.output_in_place), - log_verbose = subtensor.register.get('verbose', bittensor.defaults.subtensor.register.verbose), + **registration_args, ) return wallet \ No newline at end of file From f466485b2750ced98aada2a55d1e5e332446a4f2 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 01:59:57 -0400 Subject: [PATCH 23/40] modify naming --- bittensor/utils/__init__.py | 2 +- bittensor/utils/registration.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 42f313389e..0c841c9bd9 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -10,7 +10,7 @@ from substrateinterface.utils import ss58 from openwallet.utils import * -from .registration import create_pow +from .registration import create_pow, __reregister_wallet as reregister RAOPERTAO = 1e9 U16_MAX = 65535 diff --git a/bittensor/utils/registration.py b/bittensor/utils/registration.py index 967aea36a2..1b0de78b4f 100644 --- a/bittensor/utils/registration.py +++ b/bittensor/utils/registration.py @@ -872,7 +872,7 @@ def create_pow( return solution -def reregister( +def __reregister_wallet( netuid: int, wallet: 'bittensor.Wallet', subtensor: 'bittensor.Subtensor', From 64bbb5d976ea6fd27ffa68b4585c6feebba06687 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 02:10:01 -0400 Subject: [PATCH 24/40] add is_hotkey_registered --- bittensor/_subtensor/subtensor_impl.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bittensor/_subtensor/subtensor_impl.py b/bittensor/_subtensor/subtensor_impl.py index 8ec3477b5e..b0bfde4f43 100644 --- a/bittensor/_subtensor/subtensor_impl.py +++ b/bittensor/_subtensor/subtensor_impl.py @@ -1121,8 +1121,11 @@ def is_hotkey_registered_any( self, hotkey_ss58: str, block: Optional[int] = Non def is_hotkey_registered_on_subnet( self, hotkey_ss58: str, netuid: int, block: Optional[int] = None) -> bool: return self.get_uid_for_hotkey_on_subnet( hotkey_ss58, netuid, block ) != None - def is_hotkey_registered( self, hotkey_ss58: str, netuid: int, block: Optional[int] = None) -> bool: - return self.get_uid_for_hotkey_on_subnet( hotkey_ss58, netuid, block ) != None + def is_hotkey_registered( self, hotkey_ss58: str, netuid: Optional[int] = None, block: Optional[int] = None) -> bool: + if netuid == None: + return self.is_hotkey_registered_any( hotkey_ss58, block ) + else: + return self.is_hotkey_registered_on_subnet( hotkey_ss58, netuid, block ) def get_uid_for_hotkey_on_subnet( self, hotkey_ss58: str, netuid: int, block: Optional[int] = None) -> Optional[int]: return self.query_subtensor( 'Uids', block, [ netuid, hotkey_ss58 ] ).value From aedd7341378244aba28da76a1185849f5d414d32 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 02:10:16 -0400 Subject: [PATCH 25/40] use new function --- bittensor/_subtensor/extrinsics/registration.py | 16 +++++++++++++--- bittensor/utils/registration.py | 10 ++++++++-- bittensor/utils/registratrion_old.py | 10 +++++++--- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/bittensor/_subtensor/extrinsics/registration.py b/bittensor/_subtensor/extrinsics/registration.py index 27d7959e47..49d1645dab 100644 --- a/bittensor/_subtensor/extrinsics/registration.py +++ b/bittensor/_subtensor/extrinsics/registration.py @@ -113,7 +113,11 @@ def register_extrinsic ( # pow failed if not pow_result: # might be registered already on this subnet - if (wallet.is_registered( subtensor = subtensor, netuid = netuid )): + is_registered = subtensor.is_hotkey_registered( + netuid = netuid, + hotkey_ss58 = wallet.hotkey.ss58_address, + ) + if is_registered: bittensor.__console__.print(f":white_heavy_check_mark: [green]Already registered on netuid:{netuid}[/green]") return True @@ -143,7 +147,10 @@ def register_extrinsic ( # Successful registration, final check for neuron and pubkey else: bittensor.__console__.print(":satellite: Checking Balance...") - is_registered = wallet.is_registered( subtensor = subtensor, netuid = netuid ) + is_registered = subtensor.is_hotkey_registered( + netuid = netuid, + hotkey_ss58 = wallet.hotkey.ss58_address, + ) if is_registered: bittensor.__console__.print(":white_heavy_check_mark: [green]Registered[/green]") return True @@ -239,7 +246,10 @@ def burned_register_extrinsic ( new_balance = subtensor.get_balance( wallet.coldkeypub.ss58_address, block = block ) bittensor.__console__.print("Balance:\n [blue]{}[/blue] :arrow_right: [green]{}[/green]".format( old_balance, new_balance )) - is_registered = wallet.is_registered( subtensor = subtensor, netuid = netuid ) + is_registered = subtensor.is_hotkey_registered( + netuid = netuid, + hotkey_ss58 = wallet.hotkey.ss58_address, + ) if is_registered: bittensor.__console__.print(":white_heavy_check_mark: [green]Registered[/green]") return True diff --git a/bittensor/utils/registration.py b/bittensor/utils/registration.py index 1b0de78b4f..8def93288f 100644 --- a/bittensor/utils/registration.py +++ b/bittensor/utils/registration.py @@ -444,7 +444,10 @@ def _solve_for_difficulty_fast( subtensor, wallet: 'bittensor.Wallet', netuid: i hash_rates = [0] * n_samples # The last n true hash_rates weights = [alpha_ ** i for i in range(n_samples)] # weights decay by alpha - while not wallet.is_registered(netuid = netuid, subtensor = subtensor): + while not subtensor.is_hotkey_registered( + netuid = netuid, + hotkey_ss58 = wallet.hotkey.ss58_address, + ): # Wait until a solver finds a solution try: solution = solution_queue.get(block=True, timeout=0.25) @@ -733,7 +736,10 @@ def _solve_for_difficulty_fast_cuda( subtensor: 'bittensor.Subtensor', wallet: ' weights = [alpha_ ** i for i in range(n_samples)] # weights decay by alpha solution = None - while not wallet.is_registered(netuid = netuid, subtensor = subtensor): + while not subtensor.is_hotkey_registered( + netuid = netuid, + hotkey_ss58 = wallet.hotkey.ss58_address, + ): # Wait until a solver finds a solution try: solution = solution_queue.get(block=True, timeout=0.15) diff --git a/bittensor/utils/registratrion_old.py b/bittensor/utils/registratrion_old.py index 384503db93..bb35b549cb 100644 --- a/bittensor/utils/registratrion_old.py +++ b/bittensor/utils/registratrion_old.py @@ -377,7 +377,7 @@ def update( self, stats: RegistrationStatistics, verbose: bool = False ) -> None self.console.log( self.get_status_message(stats, verbose=verbose), ) -def solve_for_difficulty_fast( subtensor, wallet, output_in_place: bool = True, num_processes: Optional[int] = None, update_interval: Optional[int] = None, n_samples: int = 10, alpha_: float = 0.80, log_verbose: bool = False ) -> Optional[POWSolution]: +def solve_for_difficulty_fast( subtensor: 'bittensor.Subtensor', wallet, output_in_place: bool = True, num_processes: Optional[int] = None, update_interval: Optional[int] = None, n_samples: int = 10, alpha_: float = 0.80, log_verbose: bool = False ) -> Optional[POWSolution]: """ Solves the POW for registration using multiprocessing. Args: @@ -474,7 +474,9 @@ def solve_for_difficulty_fast( subtensor, wallet, output_in_place: bool = True, hash_rates = [0] * n_samples # The last n true hash_rates weights = [alpha_ ** i for i in range(n_samples)] # weights decay by alpha - while not wallet.is_registered(subtensor): + while not subtensor.is_hotkey_registered( + hotkey_ss58 = wallet.hotkey.ss58_address, + ): # Wait until a solver finds a solution try: solution = solution_queue.get(block=True, timeout=0.25) @@ -732,7 +734,9 @@ def solve_for_difficulty_fast_cuda( subtensor: 'bittensor.Subtensor', wallet: 'b weights = [alpha_ ** i for i in range(n_samples)] # weights decay by alpha solution = None - while not wallet.is_registered(subtensor): + while not subtensor.is_hotkey_registered( + hotkey_ss58 = wallet.hotkey.ss58_address, + ): # Wait until a solver finds a solution try: solution = solution_queue.get(block=True, timeout=0.15) From 450381c2b7c29399c5050a65224ce7657d9136f0 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 02:18:50 -0400 Subject: [PATCH 26/40] fix patch --- .../bittensor_tests/utils/test_utils.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/unit_tests/bittensor_tests/utils/test_utils.py b/tests/unit_tests/bittensor_tests/utils/test_utils.py index c000e4d14f..a9f71e33db 100644 --- a/tests/unit_tests/bittensor_tests/utils/test_utils.py +++ b/tests/unit_tests/bittensor_tests/utils/test_utils.py @@ -682,7 +682,7 @@ def test_wallet_reregister_reregister_false(self): class MockException(Exception): pass - with patch('bittensor.Subtensor.register', side_effect=MockException) as mock_register: + with patch('bittensor._subtensor.extrinsics.registration.register_extrinsic', side_effect=MockException) as mock_register: with pytest.raises(SystemExit): # should exit because it's not registered bittensor.utils.reregister( wallet = mock_wallet, @@ -709,7 +709,7 @@ class MockException(Exception): hotkey_ss58 = mock_wallet.hotkey.ss58_address, )) - with patch('bittensor.Subtensor.register', side_effect=MockException) as mock_register: + with patch('bittensor._subtensor.subtensor_impl.register_extrinsic', side_effect=MockException) as mock_register: bittensor.utils.reregister( wallet = mock_wallet, subtensor = self._mock_subtensor, @@ -735,7 +735,7 @@ class MockException(Exception): hotkey_ss58 = mock_wallet.hotkey.ss58_address, )) - with patch('bittensor.Subtensor.register', side_effect=MockException) as mock_register: + with patch('bittensor._subtensor.subtensor_impl.register_extrinsic', side_effect=MockException) as mock_register: bittensor.utils.reregister( wallet = mock_wallet, subtensor = self._mock_subtensor, @@ -752,7 +752,7 @@ def test_wallet_reregister_no_params(self): class MockException(Exception): pass - with patch('bittensor.Subtensor.register', side_effect=MockException) as mock_register: + with patch('bittensor._subtensor.subtensor_impl.register_extrinsic', side_effect=MockException) as mock_register: # Should be able to set without argument with pytest.raises(MockException): bittensor.utils.reregister( @@ -771,7 +771,7 @@ def test_wallet_reregister_use_cuda_flag_true(self): class MockException(Exception): pass - with patch('bittensor.Subtensor.register', side_effect=MockException) as mock_register: + with patch('bittensor._subtensor.subtensor_impl.register_extrinsic', side_effect=MockException) as mock_register: # Should be able to set without argument with pytest.raises(MockException): bittensor.utils.reregister( @@ -787,6 +787,7 @@ class MockException(Exception): _, kwargs = call_args mock_register.assert_called_once() + self.assertIn('cuda', kwargs) self.assertEqual(kwargs['cuda'], True) def test_wallet_reregister_use_cuda_flag_false(self): @@ -795,7 +796,7 @@ def test_wallet_reregister_use_cuda_flag_false(self): class MockException(Exception): pass - with patch('bittensor.Subtensor.register', side_effect=MockException) as mock_register: + with patch('bittensor._subtensor.subtensor_impl.register_extrinsic', side_effect=MockException) as mock_register: # Should be able to set without argument with pytest.raises(MockException): bittensor.utils.reregister( @@ -819,7 +820,7 @@ def test_wallet_reregister_use_cuda_flag_not_specified_false(self): class MockException(Exception): pass - with patch('bittensor.Subtensor.register', side_effect=MockException) as mock_register: + with patch('bittensor._subtensor.subtensor_impl.register_extrinsic', side_effect=MockException) as mock_register: # Should be able to set without argument with pytest.raises(MockException): bittensor.utils.reregister( From b41080b10397e0d2d6ac39186368b796e5a375d3 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 02:19:24 -0400 Subject: [PATCH 27/40] arg is cuda (typo) --- tests/unit_tests/bittensor_tests/utils/test_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit_tests/bittensor_tests/utils/test_utils.py b/tests/unit_tests/bittensor_tests/utils/test_utils.py index a9f71e33db..4a370862e4 100644 --- a/tests/unit_tests/bittensor_tests/utils/test_utils.py +++ b/tests/unit_tests/bittensor_tests/utils/test_utils.py @@ -779,7 +779,7 @@ class MockException(Exception): subtensor = self._mock_subtensor, netuid = 3, dev_id = 0, - use_cuda = True, + cuda = True, reregister = True, ) @@ -804,7 +804,7 @@ class MockException(Exception): subtensor = self._mock_subtensor, netuid = 3, dev_id = 0, - use_cuda = False, + cuda = False, reregister = True, ) From d75afc56b6570c1b619ba49f98b1d70127d8bd92 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 02:20:32 -0400 Subject: [PATCH 28/40] specify reregister true --- tests/unit_tests/bittensor_tests/utils/test_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit_tests/bittensor_tests/utils/test_utils.py b/tests/unit_tests/bittensor_tests/utils/test_utils.py index 4a370862e4..ad0427c28d 100644 --- a/tests/unit_tests/bittensor_tests/utils/test_utils.py +++ b/tests/unit_tests/bittensor_tests/utils/test_utils.py @@ -814,7 +814,7 @@ class MockException(Exception): mock_register.assert_called_once() self.assertEqual(kwargs['cuda'], False) - def test_wallet_reregister_use_cuda_flag_not_specified_false(self): + def test_wallet_reregister_cuda_arg_not_specified_should_be_false(self): mock_wallet = generate_wallet() class MockException(Exception): @@ -828,6 +828,7 @@ class MockException(Exception): subtensor = self._mock_subtensor, netuid = 3, dev_id = 0, + reregister = True, ) call_args = mock_register.call_args From 587e08da893c5c6cc3b25935c5a7d61603cdf689 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 02:28:56 -0400 Subject: [PATCH 29/40] use generate wallet --- tests/helpers.py | 20 +++++++ tests/integration_tests/test_cli.py | 47 ++++++++------- tests/unit_tests/bittensor_tests/test_axon.py | 11 +++- .../bittensor_tests/utils/test_utils.py | 57 ++++++++++++------- 4 files changed, 91 insertions(+), 44 deletions(-) diff --git a/tests/helpers.py b/tests/helpers.py index 5e68aeb3b7..771880dfa3 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -21,6 +21,8 @@ from rich.console import Console from rich.text import Text +from tests.mocks.wallet_mock import MockWallet + from Crypto.Hash import keccak class CLOSE_IN_VALUE(): @@ -127,6 +129,24 @@ def get_mock_neuron_by_uid( uid: int, **kwargs ) -> NeuronInfo: **kwargs ) +def get_mock_wallet(coldkey: "Keypair" = None, hotkey: "Keypair" = None): + wallet = MockWallet( + name = 'mock_wallet', + hotkey = 'mock', + path = '/tmp/mock_wallet', + ) + + if not coldkey: + coldkey = Keypair.create_from_mnemonic(Keypair.generate_mnemonic()) + if not hotkey: + hotkey = Keypair.create_from_mnemonic(Keypair.generate_mnemonic()) + + wallet.set_coldkey(coldkey, encrypt=False, overwrite=True) + wallet.set_coldkeypub(coldkey, encrypt=False, overwrite=True) + wallet.set_hotkey(hotkey, encrypt=False, overwrite=True) + + return wallet + class MockStatus: def __enter__(self): return self diff --git a/tests/integration_tests/test_cli.py b/tests/integration_tests/test_cli.py index b7626f0a66..14f82549e4 100644 --- a/tests/integration_tests/test_cli.py +++ b/tests/integration_tests/test_cli.py @@ -30,25 +30,10 @@ import bittensor from bittensor.utils.balance import Balance -from tests.helpers import MockConsole, get_mock_keypair +from tests.helpers import MockConsole, get_mock_keypair, get_mock_wallet as generate_wallet from bittensor._subtensor.subtensor_mock import MockSubtensor -def generate_wallet(coldkey: "Keypair" = None, hotkey: "Keypair" = None): - _config = TestCLIWithNetworkAndConfig.construct_config() - wallet = bittensor.wallet(_config, _mock=True).create() - - if not coldkey: - coldkey = Keypair.create_from_mnemonic(Keypair.generate_mnemonic()) - if not hotkey: - hotkey = Keypair.create_from_mnemonic(Keypair.generate_mnemonic()) - - wallet.set_coldkey(coldkey, encrypt=False, overwrite=True) - wallet.set_coldkeypub(coldkey, encrypt=False, overwrite=True) - wallet.set_hotkey(hotkey, encrypt=False, overwrite=True) - - return wallet - _subtensor_mock: MockSubtensor = bittensor.subtensor( network = 'mock', _mock = True ) def setUpModule(): @@ -1958,7 +1943,11 @@ def test_register(self): config.subtensor.register.update_interval = 50_000 config.no_prompt = True - mock_wallet = generate_wallet() + mock_wallet = generate_wallet( + hotkey = get_mock_keypair( + 100, self.id() + ) + ) class MockException(Exception): pass @@ -1983,7 +1972,11 @@ def test_recycle_register(self): config.command = "recycle_register" config.no_prompt = True - mock_wallet = generate_wallet() + mock_wallet = generate_wallet( + hotkey = get_mock_keypair( + 100, self.id() + ) + ) # Give the wallet some balance for burning success, err = _subtensor_mock.force_set_balance( @@ -2019,7 +2012,11 @@ def test_stake(self): subtensor = bittensor.subtensor(config) - mock_wallet = generate_wallet() + mock_wallet = generate_wallet( + hotkey = get_mock_keypair( + 100, self.id() + ) + ) # Register the hotkey and give it some balance _subtensor_mock.force_register_neuron( @@ -2191,8 +2188,16 @@ def test_delegate(self): """ Test delegate add command """ - mock_wallet = generate_wallet() - delegate_wallet = generate_wallet() + mock_wallet = generate_wallet( + hotkey = get_mock_keypair( + 100, self.id() + ) + ) + delegate_wallet = generate_wallet( + hotkey = get_mock_keypair( + 100, self.id() + ) + ) # register the wallet diff --git a/tests/unit_tests/bittensor_tests/test_axon.py b/tests/unit_tests/bittensor_tests/test_axon.py index 74c5bf06b3..679dee88b2 100644 --- a/tests/unit_tests/bittensor_tests/test_axon.py +++ b/tests/unit_tests/bittensor_tests/test_axon.py @@ -26,6 +26,7 @@ import bittensor from bittensor.utils.test_utils import get_random_unused_port +from tests.helpers import get_mock_wallet, get_mock_keypair def gen_nonce(): return f"{time.monotonic_ns()}" @@ -53,11 +54,17 @@ def is_port_in_use(port): class TestAxon(unittest.TestCase): @classmethod def setUpClass(cls) -> None: - cls.wallet = wallet = bittensor.wallet.mock() + cls.wallet = wallet = get_mock_wallet( + coldkey = get_mock_keypair(0, cls.__name__), + hotkey= get_mock_keypair(100 + 0, cls.__name__), + ) cls.axon = bittensor.axon( wallet = wallet, metagraph = None ) - cls.sender_wallet = bittensor.wallet.mock() + cls.sender_wallet = get_mock_wallet( + coldkey = get_mock_keypair(1, cls.__name__), + hotkey= get_mock_keypair(100 + 1, cls.__name__), + ) def test_axon_start(self): diff --git a/tests/unit_tests/bittensor_tests/utils/test_utils.py b/tests/unit_tests/bittensor_tests/utils/test_utils.py index ad0427c28d..fc8ba90d4f 100644 --- a/tests/unit_tests/bittensor_tests/utils/test_utils.py +++ b/tests/unit_tests/bittensor_tests/utils/test_utils.py @@ -27,6 +27,7 @@ from bittensor._subtensor.subtensor_mock import MockSubtensor from tests.mocks.wallet_mock import MockWallet +from tests.helpers import get_mock_wallet as generate_wallet, get_mock_keypair @fixture(scope="function") @@ -69,20 +70,6 @@ def select_port(): port = random.randrange(1000, 65536, 5) return port -def generate_wallet(coldkey : 'Keypair' = None, hotkey: 'Keypair' = None): - wallet = MockWallet( _mock = True, name = 'mock', path = '/tmp/', hotkey = 'mock' ) - - if not coldkey: - coldkey = Keypair.create_from_mnemonic(Keypair.generate_mnemonic()) - if not hotkey: - hotkey = Keypair.create_from_mnemonic(Keypair.generate_mnemonic()) - - wallet.set_coldkey(coldkey, encrypt=False, overwrite=True) - wallet.set_coldkeypub(coldkey, encrypt=False, overwrite=True) - wallet.set_hotkey(hotkey, encrypt=False, overwrite=True) - - return wallet - def setup_subtensor( port:int ): chain_endpoint = "localhost:{}".format(port) subtensor = bittensor.subtensor( @@ -677,7 +664,11 @@ def _do_setup_subnet(cls): ) def test_wallet_reregister_reregister_false(self): - mock_wallet = generate_wallet() + mock_wallet = generate_wallet( + hotkey = get_mock_keypair( + 100, self.id() + ) + ) class MockException(Exception): pass @@ -694,7 +685,11 @@ class MockException(Exception): mock_register.assert_not_called() # should not call register def test_wallet_reregister_reregister_false_and_registered_already(self): - mock_wallet = generate_wallet() + mock_wallet = generate_wallet( + hotkey = get_mock_keypair( + 100, self.id() + ) + ) class MockException(Exception): pass @@ -720,7 +715,11 @@ class MockException(Exception): mock_register.assert_not_called() # should not call register def test_wallet_reregister_reregister_true_and_registered_already(self): - mock_wallet = generate_wallet() + mock_wallet = generate_wallet( + hotkey = get_mock_keypair( + 100, self.id() + ) + ) class MockException(Exception): pass @@ -747,7 +746,11 @@ class MockException(Exception): def test_wallet_reregister_no_params(self): - mock_wallet = generate_wallet() + mock_wallet = generate_wallet( + hotkey = get_mock_keypair( + 100, self.id() + ) + ) class MockException(Exception): pass @@ -766,7 +769,11 @@ class MockException(Exception): mock_register.assert_called_once() # should call register once def test_wallet_reregister_use_cuda_flag_true(self): - mock_wallet = generate_wallet() + mock_wallet = generate_wallet( + hotkey = get_mock_keypair( + 100, self.id() + ) + ) class MockException(Exception): pass @@ -791,7 +798,11 @@ class MockException(Exception): self.assertEqual(kwargs['cuda'], True) def test_wallet_reregister_use_cuda_flag_false(self): - mock_wallet = generate_wallet() + mock_wallet = generate_wallet( + hotkey = get_mock_keypair( + 100, self.id() + ) + ) class MockException(Exception): pass @@ -815,7 +826,11 @@ class MockException(Exception): self.assertEqual(kwargs['cuda'], False) def test_wallet_reregister_cuda_arg_not_specified_should_be_false(self): - mock_wallet = generate_wallet() + mock_wallet = generate_wallet( + hotkey = get_mock_keypair( + 100, self.id() + ) + ) class MockException(Exception): pass From 2afb79cf4901660513f07e6d21b2c932a21275c7 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 02:43:11 -0400 Subject: [PATCH 30/40] get mock wallet during tests --- tests/integration_tests/test_prometheus.py | 3 +- .../test_subtensor_integration.py | 32 +++++++++++++------ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/tests/integration_tests/test_prometheus.py b/tests/integration_tests/test_prometheus.py index 1be071397a..992ef5cfd2 100644 --- a/tests/integration_tests/test_prometheus.py +++ b/tests/integration_tests/test_prometheus.py @@ -4,6 +4,7 @@ import unittest from unittest.mock import MagicMock, patch from bittensor._subtensor.subtensor_mock import MockSubtensor +from tests.helpers import get_mock_wallet _subtensor_mock: MockSubtensor = bittensor.subtensor( network = 'mock', _mock = True ) @@ -23,7 +24,7 @@ class TestPrometheus(unittest.TestCase): def setUp(self): self.subtensor = bittensor.subtensor(network = 'mock') - self.wallet = bittensor.wallet.mock() + self.wallet = get_mock_wallet() def test_init_prometheus_success(self): with patch.object(self.subtensor, '_do_serve_prometheus', return_value = (True, None)): diff --git a/tests/integration_tests/test_subtensor_integration.py b/tests/integration_tests/test_subtensor_integration.py index f973a0c722..87bc8b98ef 100644 --- a/tests/integration_tests/test_subtensor_integration.py +++ b/tests/integration_tests/test_subtensor_integration.py @@ -28,7 +28,7 @@ from bittensor.utils.balance import Balance from substrateinterface import Keypair from bittensor._subtensor.subtensor_mock import MockSubtensor -from tests.helpers import get_mock_hotkey, get_mock_coldkey, MockConsole, get_mock_keypair +from tests.helpers import get_mock_hotkey, get_mock_coldkey, MockConsole, get_mock_keypair, get_mock_wallet class TestSubtensor(unittest.TestCase): _mock_console_patcher = None @@ -36,7 +36,10 @@ class TestSubtensor(unittest.TestCase): subtensor: MockSubtensor def setUp(self): - self.wallet = bittensor.wallet(_mock=True) + self.wallet = get_mock_wallet( + hotkey = get_mock_keypair(0, self.id()), + coldkey = get_mock_keypair(1, self.id()) + ) self.balance = Balance.from_tao(1000) self.mock_neuron = MagicMock() # NOTE: this might need more sophistication self.subtensor = bittensor.subtensor( network = 'mock' ) # own instance per test @@ -378,8 +381,11 @@ def test_registration_multiprocessed_already_registered( self ): # patch time queue get to raise Empty exception with patch('multiprocessing.queues.Queue.get_nowait', side_effect=QueueEmpty) as mock_queue_get_nowait: - wallet = bittensor.wallet(_mock=True) - wallet.is_registered = MagicMock( side_effect=is_registered_return_values ) + wallet = get_mock_wallet( + hotkey = get_mock_keypair(0, self.id()), + coldkey = get_mock_keypair(1, self.id()) + ) + self.subtensor.is_hotkey_registered = MagicMock( side_effect=is_registered_return_values ) self.subtensor.difficulty= MagicMock(return_value=1) self.subtensor.get_neuron_for_pubkey_and_subnet = MagicMock( side_effect=mock_neuron ) @@ -395,7 +401,7 @@ def test_registration_multiprocessed_already_registered( self ): # calls until True and once again before exiting subtensor class # This assertion is currently broken when difficulty is too low - assert wallet.is_registered.call_count == workblocks_before_is_registered + 2 + assert self.subtensor.is_hotkey_registered.call_count == workblocks_before_is_registered + 2 def test_registration_partly_failed( self ): do_pow_register_mock = MagicMock( side_effect = [(False, 'Failed'), (False, 'Failed'), (True, None)]) @@ -408,8 +414,12 @@ def is_registered_side_effect(*args, **kwargs): with patch('bittensor.Subtensor.get_neuron_for_pubkey_and_subnet', return_value = bittensor.NeuronInfo._null_neuron()): with patch('bittensor.Subtensor.difficulty'): - wallet = bittensor.wallet(_mock=True) - wallet.is_registered = MagicMock(side_effect=is_registered_side_effect) + wallet = get_mock_wallet( + hotkey = get_mock_keypair(0, self.id()), + coldkey = get_mock_keypair(1, self.id()) + ) + + self.subtensor.is_hotkey_registered = MagicMock(side_effect=is_registered_side_effect) self.subtensor.difficulty = MagicMock(return_value=1) self.subtensor.get_current_block = MagicMock(side_effect=current_block) @@ -425,8 +435,12 @@ def test_registration_failed( self ): mock_neuron.is_null = True with patch('bittensor._subtensor.extrinsics.registration.create_pow', return_value=None) as mock_create_pow: - wallet = bittensor.wallet(_mock=True) - wallet.is_registered = MagicMock( side_effect=is_registered_return_values ) + wallet = get_mock_wallet( + hotkey = get_mock_keypair(0, self.id()), + coldkey = get_mock_keypair(1, self.id()) + ) + + self.subtensor.is_hotkey_registered = MagicMock(side_effect=is_registered_return_values) self.subtensor.get_current_block = MagicMock(side_effect=current_block) self.subtensor.get_neuron_for_pubkey_and_subnet = MagicMock( return_value=mock_neuron ) From c7a5d8c41e0faf5057e5b5c7762d76dcb90757e1 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 02:43:24 -0400 Subject: [PATCH 31/40] extract get ss58 format util --- bittensor/utils/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 0c841c9bd9..88b09a33a1 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -74,9 +74,6 @@ def version_checking(): if latest_version_as_int > bittensor.__version_as_int__: print('\u001b[33mBittensor Version: Current {}/Latest {}\nPlease update to the latest version at your earliest convenience\u001b[0m'.format(bittensor.__version__,latest_version)) -def get_ss58_format( ss58_address: str ) -> int: - """Returns the ss58 format of the given ss58 address.""" - return ss58.get_ss58_format( ss58_address ) def strtobool_with_default( default: bool ) -> Callable[[str], Union[bool, Literal['==SUPRESS==']]]: """ From 0608f7df3e7787fa1b42990cd5d10be975ec72af Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 02:43:32 -0400 Subject: [PATCH 32/40] remove _mock arg --- tests/mocks/wallet_mock.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/mocks/wallet_mock.py b/tests/mocks/wallet_mock.py index 0715abadc9..5e9fff9ce1 100644 --- a/tests/mocks/wallet_mock.py +++ b/tests/mocks/wallet_mock.py @@ -30,7 +30,6 @@ class MockWallet(openwallet.Wallet): """ def __init__( self, - _mock:bool, **kwargs, ): r""" Init bittensor wallet object containing a hot and coldkey. @@ -40,7 +39,7 @@ def __init__( """ super().__init__(**kwargs) # For mocking. - self._is_mock = _mock + self._is_mock = True self._mocked_coldkey_keyfile = None self._mocked_hotkey_keyfile = None From 191f417f88fefe881428ef392c5ae1b8fa47955c Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 03:01:42 -0400 Subject: [PATCH 33/40] extract tests --- tests/integration_tests/test_keyfile.py | 159 ------------- .../bittensor_tests/test_keypair.py | 213 ------------------ 2 files changed, 372 deletions(-) delete mode 100644 tests/integration_tests/test_keyfile.py delete mode 100644 tests/unit_tests/bittensor_tests/test_keypair.py diff --git a/tests/integration_tests/test_keyfile.py b/tests/integration_tests/test_keyfile.py deleted file mode 100644 index 39ccfe0616..0000000000 --- a/tests/integration_tests/test_keyfile.py +++ /dev/null @@ -1,159 +0,0 @@ -# The MIT License (MIT) -# Copyright © 2021 Yuma Rao - -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the “Software”), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. - -import os -import shutil -import time -# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. -import unittest -import unittest.mock as mock - -import pytest - -import bittensor - - -class TestKeyFiles(unittest.TestCase): - - def setUp(self) -> None: - self.root_path = f"/tmp/pytest{time.time()}" - os.makedirs(self.root_path) - - self.create_keyfile() - - def tearDown(self) -> None: - shutil.rmtree(self.root_path) - - def create_keyfile(self): - keyfile = bittensor.keyfile(path=os.path.join(self.root_path, "keyfile")) - - mnemonic = bittensor.Keypair.generate_mnemonic(12) - alice = bittensor.Keypair.create_from_mnemonic(mnemonic) - keyfile.set_keypair(alice, encrypt=True, overwrite=True, password='thisisafakepassword') - - bob = bittensor.Keypair.create_from_uri('/Bob') - keyfile.set_keypair(bob, encrypt=True, overwrite=True, password='thisisafakepassword') - - return keyfile - - def test_create(self): - keyfile = bittensor.keyfile(path=os.path.join(self.root_path, "keyfile")) - - mnemonic = bittensor.Keypair.generate_mnemonic( 12 ) - alice = bittensor.Keypair.create_from_mnemonic(mnemonic) - keyfile.set_keypair(alice, encrypt=True, overwrite=True, password = 'thisisafakepassword') - assert keyfile.is_readable() - assert keyfile.is_writable() - assert keyfile.is_encrypted() - keyfile.decrypt( password = 'thisisafakepassword' ) - assert not keyfile.is_encrypted() - keyfile.encrypt( password = 'thisisafakepassword' ) - assert keyfile.is_encrypted() - str(keyfile) - keyfile.decrypt( password = 'thisisafakepassword' ) - assert not keyfile.is_encrypted() - str(keyfile) - - assert keyfile.get_keypair( password = 'thisisafakepassword' ).ss58_address == alice.ss58_address - assert keyfile.get_keypair( password = 'thisisafakepassword' ).private_key == alice.private_key - assert keyfile.get_keypair( password = 'thisisafakepassword' ).public_key == alice.public_key - - bob = bittensor.Keypair.create_from_uri ('/Bob') - keyfile.set_keypair(bob, encrypt=True, overwrite=True, password = 'thisisafakepassword') - assert keyfile.get_keypair( password = 'thisisafakepassword' ).ss58_address == bob.ss58_address - assert keyfile.get_keypair( password = 'thisisafakepassword' ).public_key == bob.public_key - - repr(keyfile) - - def test_legacy_coldkey(self): - legacy_filename = os.path.join(self.root_path, "coldlegacy_keyfile") - keyfile = bittensor.keyfile (path = legacy_filename) - keyfile.make_dirs() - keyfile_data = b'0x32939b6abc4d81f02dff04d2b8d1d01cc8e71c5e4c7492e4fa6a238cdca3512f' - with open(legacy_filename, "wb") as keyfile_obj: - keyfile_obj.write( keyfile_data ) - assert keyfile.keyfile_data == keyfile_data - keyfile.encrypt( password = 'this is the fake password' ) - keyfile.decrypt( password = 'this is the fake password' ) - keypair_bytes = b'{"accountId": "0x32939b6abc4d81f02dff04d2b8d1d01cc8e71c5e4c7492e4fa6a238cdca3512f", "publicKey": "0x32939b6abc4d81f02dff04d2b8d1d01cc8e71c5e4c7492e4fa6a238cdca3512f", "secretPhrase": null, "secretSeed": null, "ss58Address": "5DD26kC2kxajmwfbbZmVmxhrY9VeeyR1Gpzy9i8wxLUg6zxm"}' - assert keyfile.keyfile_data == keypair_bytes - assert keyfile.get_keypair().ss58_address == "5DD26kC2kxajmwfbbZmVmxhrY9VeeyR1Gpzy9i8wxLUg6zxm" - assert "0x" + keyfile.get_keypair().public_key.hex() == "0x32939b6abc4d81f02dff04d2b8d1d01cc8e71c5e4c7492e4fa6a238cdca3512f" - - def test_validate_password(self): - from bittensor._keyfile.keyfile_impl import validate_password - assert validate_password(None) == False - assert validate_password('passw0rd') == False - assert validate_password('123456789') == False - with mock.patch('getpass.getpass',return_value='biTTensor'): - assert validate_password('biTTensor') == True - with mock.patch('getpass.getpass',return_value='biTTenso'): - assert validate_password('biTTensor') == False - - def test_decrypt_keyfile_data_legacy(self): - import base64 - - from cryptography.fernet import Fernet - from cryptography.hazmat.backends import default_backend - from cryptography.hazmat.primitives import hashes - from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC - - from bittensor._keyfile.keyfile_impl import decrypt_keyfile_data - - __SALT = b"Iguesscyborgslikemyselfhaveatendencytobeparanoidaboutourorigins" - - def __generate_key(password): - kdf = PBKDF2HMAC(algorithm=hashes.SHA256(), salt=__SALT, length=32, iterations=10000000, backend=default_backend()) - key = base64.urlsafe_b64encode(kdf.derive(password.encode())) - return key - - pw = 'fakepasssword238947239' - data = b'encrypt me!' - key = __generate_key(pw) - cipher_suite = Fernet(key) - encrypted_data = cipher_suite.encrypt(data) - - decrypted_data = decrypt_keyfile_data( encrypted_data, pw) - assert decrypted_data == data - - def test_user_interface(self): - from bittensor._keyfile.keyfile_impl import ask_password_to_encrypt - - with mock.patch('getpass.getpass', side_effect = ['pass', 'password', 'asdury3294y', 'asdury3294y']): - assert ask_password_to_encrypt() == 'asdury3294y' - - def test_overwriting(self): - from bittensor._keyfile.keyfile_impl import KeyFileError - - keyfile = bittensor.keyfile (path = os.path.join(self.root_path, "keyfile")) - alice = bittensor.Keypair.create_from_uri ('/Alice') - keyfile.set_keypair(alice, encrypt=True, overwrite=True, password = 'thisisafakepassword') - bob = bittensor.Keypair.create_from_uri ('/Bob') - - with pytest.raises(KeyFileError) as pytest_wrapped_e: - with mock.patch('builtins.input', return_value = 'n'): - keyfile.set_keypair(bob, encrypt=True, overwrite=False, password = 'thisisafakepassword') - - def test_keyfile_mock(self): - file = bittensor.keyfile( _mock = True ) - assert file.exists_on_device() - assert not file.is_encrypted() - assert file.is_readable() - assert file.data - assert file.keypair - file.set_keypair( keypair = bittensor.Keypair.create_from_mnemonic( mnemonic = bittensor.Keypair.generate_mnemonic() )) - - def test_keyfile_mock_func(self): - file = bittensor.keyfile.mock() diff --git a/tests/unit_tests/bittensor_tests/test_keypair.py b/tests/unit_tests/bittensor_tests/test_keypair.py deleted file mode 100644 index cd02a89f3c..0000000000 --- a/tests/unit_tests/bittensor_tests/test_keypair.py +++ /dev/null @@ -1,213 +0,0 @@ -# Python Substrate Interface Library -# -# Copyright 2018-2020 Stichting Polkascan (Polkascan Foundation). -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest - -from scalecodec import ScaleBytes -from substrateinterface import Keypair, KeypairType -from substrateinterface.constants import DEV_PHRASE -from substrateinterface.exceptions import ConfigurationError -from bip39 import bip39_validate - - -class KeyPairTestCase(unittest.TestCase): - - def test_generate_mnemonic(self): - mnemonic = Keypair.generate_mnemonic() - self.assertTrue(bip39_validate(mnemonic)) - - def test_invalid_mnemic(self): - mnemonic = "This is an invalid mnemonic" - self.assertFalse(bip39_validate(mnemonic)) - - def test_create_sr25519_keypair(self): - mnemonic = "old leopard transfer rib spatial phone calm indicate online fire caution review" - keypair = Keypair.create_from_mnemonic(mnemonic, ss58_format=0) - - self.assertEqual(keypair.ss58_address, "16ADqpMa4yzfmWs3nuTSMhfZ2ckeGtvqhPWCNqECEGDcGgU2") - - def test_only_provide_ss58_address(self): - - keypair = Keypair(ss58_address='16ADqpMa4yzfmWs3nuTSMhfZ2ckeGtvqhPWCNqECEGDcGgU2') - self.assertEqual("0x" + keypair.public_key.hex(), '0xe4359ad3e2716c539a1d663ebd0a51bdc5c98a12e663bb4c4402db47828c9446') - - def test_only_provide_public_key(self): - - keypair = Keypair( - public_key='0xe4359ad3e2716c539a1d663ebd0a51bdc5c98a12e663bb4c4402db47828c9446', - ss58_format=0 - ) - self.assertEqual(keypair.ss58_address, '16ADqpMa4yzfmWs3nuTSMhfZ2ckeGtvqhPWCNqECEGDcGgU2') - - def test_provide_no_ss58_address_and_public_key(self): - self.assertRaises(ValueError, Keypair) - - def test_incorrect_private_key_length_sr25519(self): - self.assertRaises( - ValueError, Keypair, private_key='0x23', ss58_address='16ADqpMa4yzfmWs3nuTSMhfZ2ckeGtvqhPWCNqECEGDcGgU2' - ) - - def test_incorrect_public_key(self): - self.assertRaises(ValueError, Keypair, public_key='0x23') - - def test_sign_and_verify(self): - mnemonic = Keypair.generate_mnemonic() - keypair = Keypair.create_from_mnemonic(mnemonic) - signature = keypair.sign("Test123") - self.assertTrue(keypair.verify("Test123", signature)) - - def test_sign_and_verify_hex_data(self): - mnemonic = Keypair.generate_mnemonic() - keypair = Keypair.create_from_mnemonic(mnemonic) - signature = keypair.sign("0x1234") - self.assertTrue(keypair.verify("0x1234", signature)) - - def test_sign_and_verify_scale_bytes(self): - mnemonic = Keypair.generate_mnemonic() - keypair = Keypair.create_from_mnemonic(mnemonic) - - data = ScaleBytes('0x1234') - - signature = keypair.sign(data) - self.assertTrue(keypair.verify(data, signature)) - - def test_sign_missing_private_key(self): - keypair = Keypair(ss58_address="5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY") - self.assertRaises(ConfigurationError, keypair.sign, "0x1234") - - def test_sign_unsupported_crypto_type(self): - keypair = Keypair.create_from_private_key( - ss58_address='16ADqpMa4yzfmWs3nuTSMhfZ2ckeGtvqhPWCNqECEGDcGgU2', - private_key='0x1f1995bdf3a17b60626a26cfe6f564b337d46056b7a1281b64c649d592ccda0a9cffd34d9fb01cae1fba61aeed184c817442a2186d5172416729a4b54dd4b84e', - crypto_type=3 - ) - self.assertRaises(ConfigurationError, keypair.sign, "0x1234") - - def test_verify_unsupported_crypto_type(self): - keypair = Keypair.create_from_private_key( - ss58_address='16ADqpMa4yzfmWs3nuTSMhfZ2ckeGtvqhPWCNqECEGDcGgU2', - private_key='0x1f1995bdf3a17b60626a26cfe6f564b337d46056b7a1281b64c649d592ccda0a9cffd34d9fb01cae1fba61aeed184c817442a2186d5172416729a4b54dd4b84e', - crypto_type=3 - ) - self.assertRaises(ConfigurationError, keypair.verify, "0x1234", '0x1234') - - def test_sign_and_verify_incorrect_signature(self): - mnemonic = Keypair.generate_mnemonic() - keypair = Keypair.create_from_mnemonic(mnemonic) - signature = "0x4c291bfb0bb9c1274e86d4b666d13b2ac99a0bacc04a4846fb8ea50bda114677f83c1f164af58fc184451e5140cc8160c4de626163b11451d3bbb208a1889f8a" - self.assertFalse(keypair.verify("Test123", signature)) - - def test_sign_and_verify_invalid_signature(self): - mnemonic = Keypair.generate_mnemonic() - keypair = Keypair.create_from_mnemonic(mnemonic) - signature = "Test" - self.assertRaises(TypeError, keypair.verify, "Test123", signature) - - def test_sign_and_verify_invalid_message(self): - mnemonic = Keypair.generate_mnemonic() - keypair = Keypair.create_from_mnemonic(mnemonic) - signature = keypair.sign("Test123") - self.assertFalse(keypair.verify("OtherMessage", signature)) - - def test_create_ed25519_keypair(self): - mnemonic = "old leopard transfer rib spatial phone calm indicate online fire caution review" - keypair = Keypair.create_from_mnemonic(mnemonic, ss58_format=0, crypto_type=KeypairType.ED25519) - - self.assertEqual(keypair.ss58_address, "16dYRUXznyhvWHS1ktUENGfNAEjCawyDzHRtN9AdFnJRc38h") - - def test_sign_and_verify_ed25519(self): - mnemonic = Keypair.generate_mnemonic() - keypair = Keypair.create_from_mnemonic(mnemonic, crypto_type=KeypairType.ED25519) - signature = keypair.sign("Test123") - - self.assertTrue(keypair.verify("Test123", signature)) - - def test_sign_and_verify_invalid_signature_ed25519(self): - mnemonic = Keypair.generate_mnemonic() - keypair = Keypair.create_from_mnemonic(mnemonic, crypto_type=KeypairType.ED25519) - signature = "0x4c291bfb0bb9c1274e86d4b666d13b2ac99a0bacc04a4846fb8ea50bda114677f83c1f164af58fc184451e5140cc8160c4de626163b11451d3bbb208a1889f8a" - self.assertFalse(keypair.verify("Test123", signature)) - - def test_unsupport_crypto_type(self): - self.assertRaises( - ValueError, Keypair.create_from_seed, - seed_hex='0xda3cf5b1e9144931?a0f0db65664aab662673b099415a7f8121b7245fb0be4143', - crypto_type=2 - ) - - def test_create_keypair_from_private_key(self): - keypair = Keypair.create_from_private_key( - ss58_address='16ADqpMa4yzfmWs3nuTSMhfZ2ckeGtvqhPWCNqECEGDcGgU2', - private_key='0x1f1995bdf3a17b60626a26cfe6f564b337d46056b7a1281b64c649d592ccda0a9cffd34d9fb01cae1fba61aeed184c817442a2186d5172416729a4b54dd4b84e' - ) - self.assertEqual("0x" + keypair.public_key.hex(), '0xe4359ad3e2716c539a1d663ebd0a51bdc5c98a12e663bb4c4402db47828c9446') - - def test_hdkd_hard_path(self): - mnemonic = 'old leopard transfer rib spatial phone calm indicate online fire caution review' - derivation_address = '5FEiH8iuDUw271xbqWTWuB6WrDjv5dnCeDX1CyHubAniXDNN' - derivation_path = '//Alice' - - derived_keypair = Keypair.create_from_uri(mnemonic + derivation_path) - - self.assertEqual(derivation_address, derived_keypair.ss58_address) - - def test_hdkd_soft_path(self): - mnemonic = 'old leopard transfer rib spatial phone calm indicate online fire caution review' - derivation_address = '5GNXbA46ma5dg19GXdiKi5JH3mnkZ8Yea3bBtZAvj7t99P9i' - derivation_path = '/Alice' - - derived_keypair = Keypair.create_from_uri(mnemonic + derivation_path) - - self.assertEqual(derivation_address, derived_keypair.ss58_address) - - def test_hdkd_default_to_dev_mnemonic(self): - derivation_address = '5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY' - derivation_path = '//Alice' - - derived_keypair = Keypair.create_from_uri(derivation_path) - - self.assertEqual(derivation_address, derived_keypair.ss58_address) - - def test_hdkd_nested_hard_soft_path(self): - derivation_address = '5CJGwWiKXSE16WJaxBdPZhWqUYkotgenLUALv7ZvqQ4TXeqf' - derivation_path = '//Bob/test' - - derived_keypair = Keypair.create_from_uri(derivation_path) - - self.assertEqual(derivation_address, derived_keypair.ss58_address) - - def test_hdkd_nested_soft_hard_path(self): - derivation_address = '5Cwc8tShrshDJUp1P1M21dKUTcYQpV9GcfSa4hUBNmMdV3Cx' - derivation_path = '/Bob//test' - - derived_keypair = Keypair.create_from_uri(derivation_path) - - self.assertEqual(derivation_address, derived_keypair.ss58_address) - - def test_hdkd_path_gt_32_bytes(self): - derivation_address = '5GR5pfZeNs1uQiSWVxZaQiZou3wdZiX894eqgvfNfHbEh7W2' - derivation_path = '//PathNameLongerThan32BytesWhichShouldBeHashed' - - derived_keypair = Keypair.create_from_uri(derivation_path) - - self.assertEqual(derivation_address, derived_keypair.ss58_address) - - def test_hdkd_unsupported_password(self): - self.assertRaises(NotImplementedError, Keypair.create_from_uri, DEV_PHRASE + '///test') - - -if __name__ == '__main__': - unittest.main() From 81063eca025e57c41b1bc3a051633dced0dfcc60 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 03:01:58 -0400 Subject: [PATCH 34/40] import from base --- tests/mocks/keyfile_mock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mocks/keyfile_mock.py b/tests/mocks/keyfile_mock.py index 198fa0facf..34c0abda4d 100644 --- a/tests/mocks/keyfile_mock.py +++ b/tests/mocks/keyfile_mock.py @@ -18,7 +18,7 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. -from openwallet.keyfile_impl import serialized_keypair_to_keyfile_data, Keyfile +from openwallet import serialized_keypair_to_keyfile_data, Keyfile from openwallet import Keypair class MockKeyfile( Keyfile ): From f2ecc0d893a6dbb9db8ed77e0a7780ff31a03f8a Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 03:02:20 -0400 Subject: [PATCH 35/40] use subtensor for get_balance --- bittensor/_cli/commands/inspect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bittensor/_cli/commands/inspect.py b/bittensor/_cli/commands/inspect.py index 3596f5a731..3b1b80549b 100644 --- a/bittensor/_cli/commands/inspect.py +++ b/bittensor/_cli/commands/inspect.py @@ -88,7 +88,7 @@ def run (cli): for wallet in tqdm( wallets ): delegates: List[Tuple(bittensor.DelegateInfo, bittensor.Balance)] = subtensor.get_delegated( coldkey_ss58=wallet.coldkeypub.ss58_address ) if not wallet.coldkeypub_file.exists_on_device(): continue - cold_balance = wallet.get_balance( subtensor = subtensor ) + cold_balance = subtensor.get_balance( wallet.coldkeypub.ss58_address ) table.add_row( wallet.name, str(cold_balance), From 03d64202f03805f974b65fa80b5f95335e1fe157 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 03:02:31 -0400 Subject: [PATCH 36/40] use new hotkey during test --- tests/integration_tests/test_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/test_cli.py b/tests/integration_tests/test_cli.py index 14f82549e4..7e16a5b4c8 100644 --- a/tests/integration_tests/test_cli.py +++ b/tests/integration_tests/test_cli.py @@ -2195,7 +2195,7 @@ def test_delegate(self): ) delegate_wallet = generate_wallet( hotkey = get_mock_keypair( - 100, self.id() + 100 + 1, self.id() ) ) @@ -2242,7 +2242,7 @@ def test_delegate(self): ) with patch( - "tests.mocks.wallet_mock.MockWallet", return_value=mock_wallet + "bittensor.wallet", return_value=mock_wallet ): # Mock wallet creation. SHOULD NOT BE REGISTERED cli = bittensor.cli( args=[ From e18f77e90bcf10e6db87faf0a91070be25620060 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 22 Jun 2023 03:07:19 -0400 Subject: [PATCH 37/40] fix mocking --- tests/unit_tests/bittensor_tests/utils/test_utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/unit_tests/bittensor_tests/utils/test_utils.py b/tests/unit_tests/bittensor_tests/utils/test_utils.py index fc8ba90d4f..bd396690f4 100644 --- a/tests/unit_tests/bittensor_tests/utils/test_utils.py +++ b/tests/unit_tests/bittensor_tests/utils/test_utils.py @@ -125,9 +125,9 @@ def test_solve_for_difficulty_fast(self): subtensor.difficulty = MagicMock( return_value=1 ) subtensor.substrate = MagicMock() subtensor.get_block_hash = MagicMock( return_value=block_hash ) + subtensor.is_hotkey_registered = MagicMock( return_value=False ) wallet = MagicMock( hotkey = Keypair.create_from_mnemonic(Keypair.generate_mnemonic()), - is_registered = MagicMock( return_value=False ) ) num_proc: int = 1 limit = int(math.pow(2,256))- 1 @@ -141,6 +141,7 @@ def test_solve_for_difficulty_fast(self): solution = bittensor.utils.registration._solve_for_difficulty_fast( subtensor, wallet, netuid = -1, num_processes=num_proc ) seal = solution.seal assert bittensor.utils.registration._seal_meets_difficulty(seal, 10, limit) + def test_solve_for_difficulty_fast_registered_already(self): # tests if the registration stops after the first block of nonces for _ in range(10): @@ -154,9 +155,9 @@ def test_solve_for_difficulty_fast_registered_already(self): subtensor.difficulty = MagicMock( return_value=int(1e10)) # set high to make solving take a long time subtensor.substrate = MagicMock() subtensor.get_block_hash = MagicMock( return_value=block_hash ) + subtensor.is_hotkey_registered = MagicMock( side_effect=is_registered_return_values ) wallet = MagicMock( hotkey = Keypair.create_from_mnemonic(Keypair.generate_mnemonic()), - is_registered = MagicMock( side_effect=is_registered_return_values ) ) # all arugments should return None to indicate an early return @@ -164,7 +165,7 @@ def test_solve_for_difficulty_fast_registered_already(self): assert solution is None # called every time until True - assert wallet.is_registered.call_count == workblocks_before_is_registered + 1 + assert subtensor.is_hotkey_registered.call_count == workblocks_before_is_registered + 1 def test_solve_for_difficulty_fast_missing_hash(self): block_hash = '0xba7ea4eb0b16dee271dbef5911838c3f359fcf598c74da65a54b919b68b67279' @@ -173,9 +174,9 @@ def test_solve_for_difficulty_fast_missing_hash(self): subtensor.difficulty = MagicMock( return_value=1 ) subtensor.substrate = MagicMock() subtensor.get_block_hash = MagicMock( side_effect= [None, None] + [block_hash]*20) + subtensor.is_hotkey_registered = MagicMock( return_value=False ) wallet = MagicMock( hotkey = Keypair.create_from_mnemonic(Keypair.generate_mnemonic()), - is_registered = MagicMock( return_value=False ) ) num_proc: int = 1 limit = int(math.pow(2,256))- 1 From 439538e65950d10958ef8c3b92c185f031c962d2 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Mon, 26 Jun 2023 17:11:08 -0400 Subject: [PATCH 38/40] add new packages to reqs file --- requirements/prod.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements/prod.txt b/requirements/prod.txt index 9068095753..a66d0a0223 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -19,7 +19,8 @@ munch==2.5.0 nest_asyncio==1.5.6 netaddr==0.8.0 numpy>=1.21.6,<=1.24.2 -openconfig==0.0.0 +open-config==0.0.0 +open-wallet==0.0.0 pandas==1.5.2 password_strength==0.0.3.post2 prometheus_client==0.14.1 From 9b1c23dae833eab53ea7671058e474f568d648b0 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Mon, 26 Jun 2023 18:41:03 -0400 Subject: [PATCH 39/40] use new package naming --- bittensor/__init__.py | 16 ++++++++-------- bittensor/utils/__init__.py | 2 +- requirements/prod.txt | 4 ++-- tests/mocks/keyfile_mock.py | 4 ++-- tests/mocks/wallet_mock.py | 12 ++++++------ 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/bittensor/__init__.py b/bittensor/__init__.py index cabe53a064..51f285ea93 100644 --- a/bittensor/__init__.py +++ b/bittensor/__init__.py @@ -131,7 +131,7 @@ def turn_console_off(): # ---- Config ---- -from openconfig import config as config +from bittensor_config import config as config # ---- LOGGING ---- # Duplicate import for ease of use. @@ -155,8 +155,8 @@ def turn_console_off(): from bittensor._cli import cli as cli from bittensor._axon import axon as axon from bittensor._axon import axon_info as axon_info -from openwallet import wallet as wallet -from openwallet import keyfile as keyfile +from bittensor_wallet import wallet as wallet +from bittensor_wallet import keyfile as keyfile from bittensor._metagraph import metagraph as metagraph from bittensor._prometheus import prometheus as prometheus from bittensor._subtensor import subtensor as subtensor @@ -169,11 +169,11 @@ def turn_console_off(): # ---- Classes ----- from bittensor._cli.cli_impl import CLI as CLI -from openconfig import Config as Config +from bittensor_config import Config as Config from bittensor._subtensor.chain_data import DelegateInfo as DelegateInfo -from openwallet import Wallet as Wallet -from openwallet import Keyfile as Keyfile -from openwallet import Keypair as Keypair +from bittensor_wallet import Wallet as Wallet +from bittensor_wallet import Keyfile as Keyfile +from bittensor_wallet import Keypair as Keypair from bittensor._subtensor.chain_data import NeuronInfo as NeuronInfo from bittensor._subtensor.chain_data import NeuronInfoLite as NeuronInfoLite from bittensor._subtensor.chain_data import PrometheusInfo as PrometheusInfo @@ -185,7 +185,7 @@ def turn_console_off(): from bittensor._ipfs.ipfs_impl import Ipfs as Ipfs # ---- Errors and Exceptions ----- -from openwallet import KeyFileError as KeyFileError +from bittensor_wallet import KeyFileError as KeyFileError from bittensor._proto.bittensor_pb2 import ForwardTextPromptingRequest from bittensor._proto.bittensor_pb2 import ForwardTextPromptingResponse diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 88b09a33a1..9f83917919 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -8,7 +8,7 @@ import scalecodec import argparse from substrateinterface.utils import ss58 -from openwallet.utils import * +from bittensor_wallet.utils import * from .registration import create_pow, __reregister_wallet as reregister diff --git a/requirements/prod.txt b/requirements/prod.txt index a66d0a0223..a7c45069d2 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -2,6 +2,8 @@ ansible_vault==2.1 argparse==1.4.0 base58==2.0.1 backoff==2.1.0 +bittensor-config==0.0.0 +bittensor-wallet==0.0.0 cryptography==39.0.0 datasets==2.12.0 fuzzywuzzy==0.18.0 @@ -19,8 +21,6 @@ munch==2.5.0 nest_asyncio==1.5.6 netaddr==0.8.0 numpy>=1.21.6,<=1.24.2 -open-config==0.0.0 -open-wallet==0.0.0 pandas==1.5.2 password_strength==0.0.3.post2 prometheus_client==0.14.1 diff --git a/tests/mocks/keyfile_mock.py b/tests/mocks/keyfile_mock.py index 34c0abda4d..92b40d8bd3 100644 --- a/tests/mocks/keyfile_mock.py +++ b/tests/mocks/keyfile_mock.py @@ -18,8 +18,8 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. -from openwallet import serialized_keypair_to_keyfile_data, Keyfile -from openwallet import Keypair +from bittensor_wallet import serialized_keypair_to_keyfile_data, Keyfile +from bittensor_wallet import Keypair class MockKeyfile( Keyfile ): """ Defines an interface to a mocked keyfile object (nothing is created on device) keypair is treated as non encrypted and the data is just the string version. diff --git a/tests/mocks/wallet_mock.py b/tests/mocks/wallet_mock.py index 5e9fff9ce1..6e7a941ea9 100644 --- a/tests/mocks/wallet_mock.py +++ b/tests/mocks/wallet_mock.py @@ -20,11 +20,11 @@ import os import bittensor -import openwallet +import bittensor_wallet from .keyfile_mock import MockKeyfile -class MockWallet(openwallet.Wallet): +class MockWallet(bittensor_wallet.Wallet): """ Mocked Version of the bittensor wallet class, meant to be used for testing """ @@ -46,7 +46,7 @@ def __init__( print("---- MOCKED WALLET INITIALIZED- ---") @property - def hotkey_file(self) -> 'openwallet.Keyfile': + def hotkey_file(self) -> 'bittensor_wallet.Keyfile': if self._is_mock: if self._mocked_hotkey_keyfile == None: self._mocked_hotkey_keyfile = MockKeyfile(path='MockedHotkey') @@ -57,7 +57,7 @@ def hotkey_file(self) -> 'openwallet.Keyfile': return bittensor.keyfile( path = hotkey_path ) @property - def coldkey_file(self) -> 'openwallet.Keyfile': + def coldkey_file(self) -> 'bittensor_wallet.Keyfile': if self._is_mock: if self._mocked_coldkey_keyfile == None: self._mocked_coldkey_keyfile = MockKeyfile(path='MockedColdkey') @@ -68,7 +68,7 @@ def coldkey_file(self) -> 'openwallet.Keyfile': return bittensor.keyfile( path = coldkey_path ) @property - def coldkeypub_file(self) -> 'openwallet.Keyfile': + def coldkeypub_file(self) -> 'bittensor_wallet.Keyfile': if self._is_mock: if self._mocked_coldkey_keyfile == None: self._mocked_coldkey_keyfile = MockKeyfile(path='MockedColdkeyPub') @@ -76,4 +76,4 @@ def coldkeypub_file(self) -> 'openwallet.Keyfile': else: wallet_path = os.path.expanduser(os.path.join(self.path, self.name)) coldkeypub_path = os.path.join(wallet_path, "coldkeypub.txt") - return openwallet.Keyfile( path = coldkeypub_path ) \ No newline at end of file + return bittensor_wallet.Keyfile( path = coldkeypub_path ) \ No newline at end of file From 0b81613b7c845c6fd1565df6052f095f50ab03b0 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Tue, 27 Jun 2023 17:48:04 -0400 Subject: [PATCH 40/40] bump wallet dep --- requirements/prod.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/prod.txt b/requirements/prod.txt index a7c45069d2..86bb295205 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -3,7 +3,7 @@ argparse==1.4.0 base58==2.0.1 backoff==2.1.0 bittensor-config==0.0.0 -bittensor-wallet==0.0.0 +bittensor-wallet==0.0.1 cryptography==39.0.0 datasets==2.12.0 fuzzywuzzy==0.18.0