From cb93237c0bdac5dd2f593370be261d1aa7530b2b Mon Sep 17 00:00:00 2001 From: opendansor Date: Fri, 24 May 2024 13:51:35 -0700 Subject: [PATCH 01/12] Add Commit weights and reveal weights function --- bittensor/cli.py | 14 + bittensor/commands/__init__.py | 1 + bittensor/commands/weights.py | 232 +++++++++++++++++ bittensor/extrinsics/commit_weights.py | 119 +++++++++ bittensor/subtensor.py | 245 +++++++++++++++++- bittensor/utils/weight_utils.py | 27 ++ .../integration_tests/test_cli_no_network.py | 2 + .../test_subtensor_integration.py | 220 +++++++++++++++- 8 files changed, 853 insertions(+), 7 deletions(-) create mode 100644 bittensor/commands/weights.py create mode 100644 bittensor/extrinsics/commit_weights.py diff --git a/bittensor/cli.py b/bittensor/cli.py index fb8f628335..49e188317c 100644 --- a/bittensor/cli.py +++ b/bittensor/cli.py @@ -68,6 +68,8 @@ VoteCommand, WalletBalanceCommand, WalletCreateCommand, + CommitWeightCommand, + RevealWeightCommand, ) # Create a console instance for CLI display. @@ -93,6 +95,9 @@ "sudos": "sudo", "i": "info", "info": "info", + "weights": "weight", + "wt": "weight", + "weight": "weight", } COMMANDS = { "subnets": { @@ -166,6 +171,15 @@ "remove": UnStakeCommand, }, }, + "weight": { + "name": "weight", + "aliases": ["wt", "weights"], + "help": "Commands for managing weight for subnets.", + "commands": { + "commit": CommitWeightCommand, + "reveal": RevealWeightCommand, + }, + }, "sudo": { "name": "sudo", "aliases": ["su", "sudos"], diff --git a/bittensor/commands/__init__.py b/bittensor/commands/__init__.py index 27883be2f8..2ccea346a4 100644 --- a/bittensor/commands/__init__.py +++ b/bittensor/commands/__init__.py @@ -91,6 +91,7 @@ WalletBalanceCommand, GetWalletHistoryCommand, ) +from .weights import CommitWeightCommand, RevealWeightCommand from .transfer import TransferCommand from .inspect import InspectCommand from .metagraph import MetagraphCommand diff --git a/bittensor/commands/weights.py b/bittensor/commands/weights.py new file mode 100644 index 0000000000..9499c60c95 --- /dev/null +++ b/bittensor/commands/weights.py @@ -0,0 +1,232 @@ +import argparse +import re + +import torch +from rich.prompt import Prompt + +import bittensor +from . import defaults + + +class CommitWeightCommand: + """ + Executes the ``commit`` command to commit weights for specific subnet on the Bittensor network. + + Usage: + The command allows committing weights for a specific subnet. Users need to specify the netuid (network unique identifier), corresponding UIDs, and weights they wish to commit. + + Optional arguments: + - ``--netuid`` (int): The netuid of the subnet for which weights are to be commited. + - ``--uids`` (str): Corresponding UIDs for the specified netuid, in comma-separated format. + - ``--weights`` (str): Corresponding weights for the specified UIDs, in comma-separated format. + + Example usage: + $ btcli wt commit --netuid 1 --uids 1,2,3,4 --weights 0.1,0.2,0.3,0.4 + + Note: + This command is used to commit weights for a specific subnet and requires the user to have the necessary permissions. + """ + + @staticmethod + def run(cli: "bittensor.cli"): + r"""Commit weights for a specific subnet.""" + try: + subtensor: "bittensor.subtensor" = bittensor.subtensor(config=cli.config, log_verbose=False) + CommitWeightCommand._run(cli, subtensor) + finally: + if "subtensor" in locals(): + subtensor.close() + bittensor.logging.debug("closing subtensor connection") + + @staticmethod + def _run(cli: "bittensor.cli", subtensor: "bittensor.subtensor"): + r"""Commit weights for a specific subnet""" + wallet = bittensor.wallet(config=cli.config) + + # Get values if not set + if not cli.config.is_set("netuid"): + cli.config.netuid = int(Prompt.ask(f"Enter netuid")) + + if not cli.config.is_set("uids"): + cli.config.uids = Prompt.ask(f"Enter UIDs (comma-separated)") + + if not cli.config.is_set("weights"): + cli.config.weights = Prompt.ask(f"Enter weights (comma-separated)") + + # Parse from string + netuid = cli.config.netuid + uids = torch.tensor( + list(map(int, re.split(r"[ ,]+", cli.config.uids))), + dtype=torch.int64 + ) + weights = torch.tensor( + list(map(int, re.split(r"[ ,]+", cli.config.weights))), + dtype=torch.float32 + ) + + # Run the commit weights operation + success, message = subtensor.commit_weights( + wallet=wallet, + netuid=netuid, + uids=uids, + weights=weights, + wait_for_inclusion=cli.config.wait_for_inclusion, + wait_for_finalization=cli.config.wait_for_finalization, + prompt=cli.config.prompt, + ) + + # Result + if success: + bittensor.__console__.print(f"Weights committed successfully") + else: + bittensor.__console__.print(f"Failed to commit weights: {message}") + + @staticmethod + def add_args(parser: argparse.ArgumentParser): + parser = parser.add_parser( + "commit", help="""Commit weights for a specific subnet.""" + ) + parser.add_argument("--netuid", dest="netuid", type=int, required=False) + parser.add_argument("--uids", dest="uids", type=str, required=False) + parser.add_argument("--weights", dest="weights", type=str, required=False) + parser.add_argument( + "--wait-for-inclusion", + dest="wait_for_inclusion", + action="store_true", + default=False, + ) + parser.add_argument( + "--wait-for-finalization", + dest="wait_for_finalization", + action="store_true", + default=True, + ) + parser.add_argument( + "--prompt", + dest="prompt", + action="store_true", + default=True, + ) + + bittensor.wallet.add_args(parser) + bittensor.subtensor.add_args(parser) + + @staticmethod + def check_config(config: "bittensor.config"): + if not config.is_set("wallet.name") and not config.no_prompt: + wallet_name = Prompt.ask("Enter wallet name", default=defaults.wallet.name) + config.wallet.name = str(wallet_name) + if not config.is_set("wallet.hotkey") and not config.no_prompt: + hotkey = Prompt.ask("Enter hotkey name", default=defaults.wallet.hotkey) + config.wallet.hotkey = str(hotkey) + + +class RevealWeightCommand: + """ + Executes the ``reveal`` command to reveal weights for a specific subnet on the Bittensor network. + Usage: + The command allows revealing weights for a specific subnet. Users need to specify the netuid (network unique identifier), corresponding UIDs, and weights they wish to reveal. + Optional arguments: + - ``--netuid`` (int): The netuid of the subnet for which weights are to be revealed. + - ``--uids`` (str): Corresponding UIDs for the specified netuid, in comma-separated format. + - ``--weights`` (str): Corresponding weights for the specified UIDs, in comma-separated format. + Example usage:: + $ btcli wt reveal --netuid 1 --uids 1,2,3,4 --weights 0.1,0.2,0.3,0.4 + Note: + This command is used to reveal weights for a specific subnet and requires the user to have the necessary permissions. + """ + + @staticmethod + def run(cli: "bittensor.cli"): + r"""Reveal weights for a specific subnet.""" + try: + subtensor: "bittensor.subtensor" = bittensor.subtensor( + config=cli.config, log_verbose=False + ) + RevealWeightCommand._run(cli, subtensor) + finally: + if "subtensor" in locals(): + subtensor.close() + bittensor.logging.debug("closing subtensor connection") + + @staticmethod + def _run(cli: "bittensor.cli", subtensor: "bittensor.subtensor"): + r"""Reveal weights for a specific subnet.""" + wallet = bittensor.wallet(config=cli.config) + + # Get values if not set. + if not cli.config.is_set("netuid"): + cli.config.netuid = int(Prompt.ask(f"Enter netuid")) + + if not cli.config.is_set("uids"): + cli.config.uids = Prompt.ask(f"Enter UIDs (comma-separated)") + + if not cli.config.is_set("weights"): + cli.config.weights = Prompt.ask(f"Enter weights (comma-separated)") + + # Parse from string + netuid = cli.config.netuid + uids = torch.tensor( + list(map(int, re.split(r"[ ,]+", cli.config.uids))), + dtype=torch.int64, + ) + weights = torch.tensor( + list(map(float, re.split(r"[ ,]+", cli.config.weights))), + dtype=torch.float32, + ) + + # Run the reveal weights operation. + success, message = subtensor.reveal_weights( + wallet=wallet, + netuid=netuid, + uids=uids, + weights=weights, + version_key=0, + wait_for_inclusion=cli.config.wait_for_inclusion, + wait_for_finalization=cli.config.wait_for_finalization, + prompt=cli.config.prompt, + ) + + if success: + bittensor.__console__.print(f"Weights revealed successfully") + else: + bittensor.__console__.print(f"Failed to reveal weights: {message}") + + @staticmethod + def add_args(parser: argparse.ArgumentParser): + parser = parser.add_parser( + "reveal", help="""Reveal weights for a specific subnet.""" + ) + parser.add_argument("--netuid", dest="netuid", type=int, required=False) + parser.add_argument("--uids", dest="uids", type=str, required=False) + parser.add_argument("--weights", dest="weights", type=str, required=False) + parser.add_argument( + "--wait-for-inclusion", + dest="wait_for_inclusion", + action="store_true", + default=False, + ) + parser.add_argument( + "--wait-for-finalization", + dest="wait_for_finalization", + action="store_true", + default=True, + ) + parser.add_argument( + "--prompt", + dest="prompt", + action="store_true", + default=True, + ) + + bittensor.wallet.add_args(parser) + bittensor.subtensor.add_args(parser) + + @staticmethod + def check_config(config: "bittensor.config"): + if not config.is_set("wallet.name") and not config.no_prompt: + wallet_name = Prompt.ask("Enter wallet name", default=defaults.wallet.name) + config.wallet.name = str(wallet_name) + if not config.is_set("wallet.hotkey") and not config.no_prompt: + hotkey = Prompt.ask("Enter hotkey name", default=defaults.wallet.hotkey) + config.wallet.hotkey = str(hotkey) diff --git a/bittensor/extrinsics/commit_weights.py b/bittensor/extrinsics/commit_weights.py new file mode 100644 index 0000000000..59fd2f2776 --- /dev/null +++ b/bittensor/extrinsics/commit_weights.py @@ -0,0 +1,119 @@ +# The MIT License (MIT) +# Copyright © 2021 Yuma Rao +# Copyright © 2023 Opentensor Foundation + +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the “Software”), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. + +# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +from typing import Tuple, List + +import bittensor + + +def commit_weights_extrinsic( + subtensor: "bittensor.subtensor", + wallet: "bittensor.wallet", + netuid: int, + commit_hash: str, + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, + prompt: bool = False, +) -> Tuple[bool, str]: + """ + Commits a hash of the neuron's weights to the Bittensor blockchain using the provided wallet. + This function is a wrapper around the `_do_commit_weights` method, handling user prompts and error messages. + Args: + subtensor (bittensor.subtensor): The subtensor instance used for blockchain interaction. + wallet (bittensor.wallet): The wallet associated with the neuron committing the weights. + netuid (int): The unique identifier of the subnet. + commit_hash (str): The hash of the neuron's weights to be committed. + wait_for_inclusion (bool, optional): Waits for the transaction to be included in a block. + wait_for_finalization (bool, optional): Waits for the transaction to be finalized on the blockchain. + prompt (bool, optional): If ``True``, prompts for user confirmation before proceeding. + Returns: + Tuple[bool, str]: ``True`` if the weight commitment is successful, False otherwise. And `msg`, a string + value describing the success or potential error. + This function provides a user-friendly interface for committing weights to the Bittensor blockchain, ensuring proper + error handling and user interaction when required. + """ + if prompt: + if not input("Would you like to commit weights? (y/n) ").lower() == "y": + return False, "User cancelled the operation." + + success, error_message = subtensor._do_commit_weights( + wallet=wallet, + netuid=netuid, + commit_hash=commit_hash, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + + if success: + bittensor.logging.info("Successfully committed weights.") + return True, "Successfully committed weights." + else: + bittensor.logging.error(f"Failed to commit weights: {error_message}") + return False, error_message + + +def reveal_weights_extrinsic( + subtensor: "bittensor.subtensor", + wallet: "bittensor.wallet", + netuid: int, + uids: List[int], + weights: List[int], + version_key: int, + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, + prompt: bool = False, +) -> Tuple[bool, str]: + """ + Reveals the weights for a specific subnet on the Bittensor blockchain using the provided wallet. + This function is a wrapper around the `_do_reveal_weights` method, handling user prompts and error messages. + Args: + subtensor (bittensor.subtensor): The subtensor instance used for blockchain interaction. + wallet (bittensor.wallet): The wallet associated with the neuron revealing the weights. + netuid (int): The unique identifier of the subnet. + uids (List[int]): List of neuron UIDs for which weights are being revealed. + weights (List[int]): List of weight values corresponding to each UID. + version_key (int): Version key for compatibility with the network. + wait_for_inclusion (bool, optional): Waits for the transaction to be included in a block. + wait_for_finalization (bool, optional): Waits for the transaction to be finalized on the blockchain. + prompt (bool, optional): If ``True``, prompts for user confirmation before proceeding. + Returns: + Tuple[bool, str]: ``True`` if the weight revelation is successful, False otherwise. And `msg`, a string + value describing the success or potential error. + This function provides a user-friendly interface for revealing weights on the Bittensor blockchain, ensuring proper + error handling and user interaction when required. + """ + if prompt: + if not input("Would you like to reveal weights? (y/n) ").lower() == "y": + return False, "User cancelled the operation." + + success, error_message = subtensor._do_reveal_weights( + wallet=wallet, + netuid=netuid, + uids=uids, + values=weights, + version_key=version_key, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + + if success: + bittensor.logging.info("Successfully revealed weights.") + return True, "Successfully revealed weights." + else: + bittensor.logging.error(f"Failed to reveal weights: {error_message}") + return False, error_message diff --git a/bittensor/subtensor.py b/bittensor/subtensor.py index 737a3b556b..2ed618fd5f 100644 --- a/bittensor/subtensor.py +++ b/bittensor/subtensor.py @@ -40,7 +40,7 @@ import bittensor from bittensor.btlogging import logging as _logger -from bittensor.utils import torch +from bittensor.utils import torch, weight_utils from .chain_data import ( NeuronInfo, DelegateInfo, @@ -86,6 +86,10 @@ publish_metadata, get_metadata, ) +from .extrinsics.commit_weights import ( + commit_weights_extrinsic, + reveal_weights_extrinsic, +) from .extrinsics.set_weights import set_weights_extrinsic from .extrinsics.staking import add_stake_extrinsic, add_stake_multiple_extrinsic from .extrinsics.transfer import transfer_extrinsic @@ -899,6 +903,245 @@ def make_substrate_call_with_retry(): return make_substrate_call_with_retry() + ################## + # Commit Weights # + ################## + def commit_weights( + self, + wallet: "bittensor.wallet", + netuid: int, + uids: torch.Tensor, + weights: torch.Tensor, + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, + prompt: bool = False, + max_retries: int = 5, + ) -> Tuple[bool, str]: + """ + Commits a hash of the neuron's weights to the Bittensor blockchain using the provided wallet. + This action serves as a commitment or snapshot of the neuron's current weight distribution. + Args: + wallet (bittensor.wallet): The wallet associated with the neuron committing the weights. + netuid (int): The unique identifier of the subnet. + wait_for_inclusion (bool, optional): Waits for the transaction to be included in a block. + wait_for_finalization (bool, optional): Waits for the transaction to be finalized on the blockchain. + prompt (bool, optional): If ``True``, prompts for user confirmation before proceeding. + max_retries (int, optional): The number of maximum attempts to commit weights. (Default: 5) + Returns: + Tuple[bool, str]: ``True`` if the weight commitment is successful, False otherwise. And `msg`, a string + value describing the success or potential error. + This function allows neurons to create a tamper-proof record of their weight distribution at a specific point in time, + enhancing transparency and accountability within the Bittensor network. + """ + uid = self.get_uid_for_hotkey_on_subnet(wallet.hotkey.ss58_address, netuid) + retries = 0 + success = False + message = "No attempt made. Perhaps it is too soon to commit weights!" + + # Generate the hash of the weights + commit_hash = weight_utils.generate_weight_hash( + who=wallet.hotkey.ss58_address, + netuid=netuid, + uids=uids.tolist(), + values=weights.tolist(), + version_key=0, + ) + + while retries < max_retries: + try: + success, message = commit_weights_extrinsic( + subtensor=self, + wallet=wallet, + netuid=netuid, + commit_hash=commit_hash, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + prompt=prompt, + ) + if success: + break + except Exception as e: + bittensor.logging.error(f"Error committing weights: {e}") + finally: + retries += 1 + + return success, message + + def _do_commit_weights( + self, + wallet: "bittensor.wallet", + netuid: int, + commit_hash: str, + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, + ) -> Tuple[bool, Optional[str]]: + """ + Internal method to send a transaction to the Bittensor blockchain, committing the hash of a neuron's weights. + This method constructs and submits the transaction, handling retries and blockchain communication. + Args: + wallet (bittensor.wallet): The wallet associated with the neuron committing the weights. + netuid (int): The unique identifier of the subnet. + commit_hash (str): The hash of the neuron's weights to be committed. + wait_for_inclusion (bool, optional): Waits for the transaction to be included in a block. + wait_for_finalization (bool, optional): Waits for the transaction to be finalized on the blockchain. + Returns: + Tuple[bool, Optional[str]]: A tuple containing a success flag and an optional error message. + This method ensures that the weight commitment is securely recorded on the Bittensor blockchain, providing a verifiable + record of the neuron's weight distribution at a specific point in time. + """ + + @retry(delay=1, tries=3, backoff=2, max_delay=4, logger=_logger) + def make_substrate_call_with_retry(): + call = self.substrate.compose_call( + call_module="SubtensorModule", + call_function="commit_weights", + call_params={ + "netuid": netuid, + "commit_hash": commit_hash, + }, + ) + extrinsic = self.substrate.create_signed_extrinsic( + call=call, + keypair=wallet.hotkey, + ) + response = self.substrate.submit_extrinsic( + extrinsic, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + + if not wait_for_finalization and not wait_for_inclusion: + return True, None + + response.process_events() + if response.is_success: + return True, None + else: + return False, response.error_message + + return make_substrate_call_with_retry() + + ################## + # Reveal Weights # + ################## + def reveal_weights( + self, + wallet: "bittensor.wallet", + netuid: int, + uids: torch.Tensor, + weights: torch.Tensor, + version_key: int, + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, + prompt: bool = False, + max_retries: int = 5, + ) -> Tuple[bool, str]: + """ + Reveals the weights for a specific subnet on the Bittensor blockchain using the provided wallet. + This action serves as a revelation of the neuron's previously committed weight distribution. + Args: + wallet (bittensor.wallet): The wallet associated with the neuron revealing the weights. + netuid (int): The unique identifier of the subnet. + uids (torch.Tensor): Tensor of neuron UIDs for which weights are being revealed. + weights (torch.Tensor): Tensor of weight values corresponding to each UID. + version_key (int): Version key for compatibility with the network. + wait_for_inclusion (bool, optional): Waits for the transaction to be included in a block. + wait_for_finalization (bool, optional): Waits for the transaction to be finalized on the blockchain. + prompt (bool, optional): If ``True``, prompts for user confirmation before proceeding. + max_retries (int, optional): The number of maximum attempts to reveal weights. (Default: 5) + Returns: + Tuple[bool, str]: ``True`` if the weight revelation is successful, False otherwise. And `msg`, a string + value describing the success or potential error. + This function allows neurons to reveal their previously committed weight distribution, ensuring transparency + and accountability within the Bittensor network. + """ + uid = self.get_uid_for_hotkey_on_subnet(wallet.hotkey.ss58_address, netuid) + retries = 0 + success = False + message = "No attempt made. Perhaps it is too soon to reveal weights!" + + while retries < max_retries: + try: + success, message = reveal_weights_extrinsic( + subtensor=self, + wallet=wallet, + netuid=netuid, + uids=uids.tolist(), + weights=weights.tolist(), + version_key=version_key, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + prompt=prompt, + ) + if success: + break + except Exception as e: + bittensor.logging.error(f"Error revealing weights: {e}") + finally: + retries += 1 + + return success, message + + def _do_reveal_weights( + self, + wallet: "bittensor.wallet", + netuid: int, + uids: List[int], + values: List[int], + version_key: int, + wait_for_inclusion: bool = False, + wait_for_finalization: bool = False, + ) -> Tuple[bool, Optional[str]]: + """ + Internal method to send a transaction to the Bittensor blockchain, revealing the weights for a specific subnet. + This method constructs and submits the transaction, handling retries and blockchain communication. + Args: + wallet (bittensor.wallet): The wallet associated with the neuron revealing the weights. + netuid (int): The unique identifier of the subnet. + uids (List[int]): List of neuron UIDs for which weights are being revealed. + values (List[int]): List of weight values corresponding to each UID. + version_key (int): Version key for compatibility with the network. + wait_for_inclusion (bool, optional): Waits for the transaction to be included in a block. + wait_for_finalization (bool, optional): Waits for the transaction to be finalized on the blockchain. + Returns: + Tuple[bool, Optional[str]]: A tuple containing a success flag and an optional error message. + This method ensures that the weight revelation is securely recorded on the Bittensor blockchain, providing transparency + and accountability for the neuron's weight distribution. + """ + + @retry(delay=2, tries=3, backoff=2, max_delay=4, logger=_logger) + def make_substrate_call_with_retry(): + call = self.substrate.compose_call( + call_module="SubtensorModule", + call_function="reveal_weights", + call_params={ + "netuid": netuid, + "uids": uids, + "values": values, + "version_key": version_key, + }, + ) + extrinsic = self.substrate.create_signed_extrinsic( + call=call, + keypair=wallet.hotkey, + ) + response = self.substrate.submit_extrinsic( + extrinsic, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + + if not wait_for_finalization and not wait_for_inclusion: + return True, None + + response.process_events() + if response.is_success: + return True, None + else: + return False, response.error_message + + return make_substrate_call_with_retry() + ################ # Registration # ################ diff --git a/bittensor/utils/weight_utils.py b/bittensor/utils/weight_utils.py index 9bd8606c9d..c78095008d 100644 --- a/bittensor/utils/weight_utils.py +++ b/bittensor/utils/weight_utils.py @@ -20,6 +20,7 @@ import numpy as np import bittensor +import hashlib from numpy.typing import NDArray from typing import Tuple, List, Union from bittensor.utils.registration import torch, use_torch, legacy_torch_api_compat @@ -341,3 +342,29 @@ def process_weights_for_netuid( bittensor.logging.debug("final_weights", normalized_weights) return non_zero_weight_uids, normalized_weights + + +def generate_weight_hash( + who: str, netuid: int, uids: List[int], values: List[int], version_key: int +) -> str: + """ + Generate a valid commit hash from the provided weights. + Args: + who (str): The account identifier. + netuid (int): The network unique identifier. + uids (List[int]): The list of UIDs. + values (List[int]): The list of weight values. + version_key (int): The version key. + Returns: + str: The generated commit hash. + """ + # Create a tuple of the input parameters + data = (who, netuid, uids, values, version_key) + + # Generate Blake2b hash of the data tuple + blake2b_hash = hashlib.blake2b(str(data).encode(), digest_size=32).digest() + + # Convert the hash to hex string and add "0x" prefix + commit_hash = "0x" + blake2b_hash.hex() + + return commit_hash diff --git a/tests/integration_tests/test_cli_no_network.py b/tests/integration_tests/test_cli_no_network.py index 18eb26cd8e..6d6d24f58a 100644 --- a/tests/integration_tests/test_cli_no_network.py +++ b/tests/integration_tests/test_cli_no_network.py @@ -427,7 +427,9 @@ def test_command_no_args(self, _, __, patched_prompt_ask): "stakes", "roots", "wallets", + "weights", "st", + "wt", "su", ] # Skip duplicate aliases ] diff --git a/tests/integration_tests/test_subtensor_integration.py b/tests/integration_tests/test_subtensor_integration.py index 845a73ee7d..ea9f2d2822 100644 --- a/tests/integration_tests/test_subtensor_integration.py +++ b/tests/integration_tests/test_subtensor_integration.py @@ -17,20 +17,18 @@ # DEALINGS IN THE SOFTWARE. import random -import socket -import os import unittest from queue import Empty as QueueEmpty from unittest.mock import MagicMock, patch -from types import SimpleNamespace + +import pytest +import torch +from substrateinterface import Keypair import bittensor from bittensor.mock import MockSubtensor -import pytest from bittensor.utils.balance import Balance -from substrateinterface import Keypair from tests.helpers import ( - _get_mock_hotkey, _get_mock_coldkey, MockConsole, _get_mock_keypair, @@ -362,6 +360,216 @@ def test_set_weights_failed(self): ) assert fail == False + def test_commit_weights(self): + weights = torch.FloatTensor([0.1, 0.2, 0.3, 0.4]) + uids = torch.tensor([1, 2, 3, 4], dtype=torch.int64) + commit_hash = bittensor.utils.weight_utils.generate_weight_hash( + who=self.wallet.hotkey.ss58_address, + netuid=3, + uids=uids.tolist(), + values=weights.tolist(), + version_key=0, + ) + + self.subtensor.commit_weights = MagicMock( + return_value=(True, "Successfully committed weights.") + ) + self.subtensor._do_commit_weights = MagicMock(return_value=(True, None)) + + success, message = self.subtensor.commit_weights( + wallet=self.wallet, + netuid=3, + uids=uids, + weights=weights, + ) + assert success == True + assert message == "Successfully committed weights." + + def test_commit_weights_inclusion(self): + weights = torch.FloatTensor([0.1, 0.2, 0.3, 0.4]) + uids = torch.tensor([1, 2, 3, 4], dtype=torch.int64) + commit_hash = bittensor.utils.weight_utils.generate_weight_hash( + who=self.wallet.hotkey.ss58_address, + netuid=1, + uids=uids.tolist(), + values=weights.tolist(), + version_key=0, + ) + + self.subtensor._do_commit_weights = MagicMock(return_value=(True, None)) + self.subtensor.commit_weights = MagicMock( + return_value=(True, "Successfully committed weights.") + ) + + success, message = self.subtensor.commit_weights( + wallet=self.wallet, + netuid=1, + uids=uids, + weights=weights, + wait_for_inclusion=True, + ) + assert success == True + assert message == "Successfully committed weights." + + def test_commit_weights_failed(self): + weights = torch.FloatTensor([0.1, 0.2, 0.3, 0.4]) + uids = torch.tensor([1, 2, 3, 4], dtype=torch.int64) + commit_hash = bittensor.utils.weight_utils.generate_weight_hash( + who=self.wallet.hotkey.ss58_address, + netuid=3, + uids=uids.tolist(), + values=weights.tolist(), + version_key=0, + ) + + self.subtensor._do_commit_weights = MagicMock( + return_value=(False, "Mock failure message") + ) + self.subtensor.commit_weights = MagicMock( + return_value=(False, "Mock failure message") + ) + + success, message = self.subtensor.commit_weights( + wallet=self.wallet, + netuid=3, + uids=uids, + weights=weights, + wait_for_inclusion=True, + ) + assert success == False + assert message == "Mock failure message" + + def test_reveal_weights(self): + weights = torch.FloatTensor([0.1, 0.2, 0.3, 0.4]) + uids = torch.tensor([1, 2, 3, 4], dtype=torch.int64) + + self.subtensor.reveal_weights = MagicMock( + return_value=(True, "Successfully revealed weights.") + ) + self.subtensor._do_reveal_weights = MagicMock(return_value=(True, None)) + + success, message = self.subtensor.reveal_weights( + wallet=self.wallet, netuid=3, uids=uids, weights=weights, version_key=0 + ) + assert success == True + assert message == "Successfully revealed weights." + + def test_reveal_weights_inclusion(self): + weights = torch.FloatTensor([0.1, 0.2, 0.3, 0.4]) + uids = torch.tensor([1, 2, 3, 4], dtype=torch.int64) + + self.subtensor._do_reveal_weights = MagicMock(return_value=(True, None)) + self.subtensor.reveal_weights = MagicMock( + return_value=(True, "Successfully revealed weights.") + ) + + success, message = self.subtensor.reveal_weights( + wallet=self.wallet, + netuid=1, + uids=uids, + weights=weights, + version_key=0, + wait_for_inclusion=True, + ) + assert success == True + assert message == "Successfully revealed weights." + + def test_reveal_weights_failed(self): + weights = torch.FloatTensor([0.1, 0.2, 0.3, 0.4]) + uids = torch.tensor([1, 2, 3, 4], dtype=torch.int64) + + self.subtensor._do_reveal_weights = MagicMock( + return_value=(False, "Mock failure message") + ) + self.subtensor.reveal_weights = MagicMock( + return_value=(False, "Mock failure message") + ) + + success, message = self.subtensor.reveal_weights( + wallet=self.wallet, + netuid=3, + uids=uids, + weights=weights, + version_key=0, + wait_for_inclusion=True, + ) + assert success == False + assert message == "Mock failure message" + + def test_commit_and_reveal_weights(self): + weights = torch.FloatTensor([0.1, 0.2, 0.3, 0.4]) + uids = torch.tensor([1, 2, 3, 4], dtype=torch.int64) + version_key = 0 + + # Mock the commit_weights and reveal_weights functions + self.subtensor.commit_weights = MagicMock( + return_value=(True, "Successfully committed weights.") + ) + self.subtensor._do_commit_weights = MagicMock(return_value=(True, None)) + self.subtensor.reveal_weights = MagicMock( + return_value=(True, "Successfully revealed weights.") + ) + self.subtensor._do_reveal_weights = MagicMock(return_value=(True, None)) + + # Commit weights + commit_success, commit_message = self.subtensor.commit_weights( + wallet=self.wallet, + netuid=3, + uids=uids, + weights=weights, + ) + assert commit_success == True + assert commit_message == "Successfully committed weights." + + # Reveal weights + reveal_success, reveal_message = self.subtensor.reveal_weights( + wallet=self.wallet, + netuid=3, + uids=uids, + weights=weights, + version_key=version_key, + ) + assert reveal_success == True + assert reveal_message == "Successfully revealed weights." + + def test_commit_and_reveal_weights_inclusion(self): + weights = torch.FloatTensor([0.1, 0.2, 0.3, 0.4]) + uids = torch.tensor([1, 2, 3, 4], dtype=torch.int64) + version_key = 0 + + # Mock the commit_weights and reveal_weights functions + self.subtensor.commit_weights = MagicMock( + return_value=(True, "Successfully committed weights.") + ) + self.subtensor._do_commit_weights = MagicMock(return_value=(True, None)) + self.subtensor.reveal_weights = MagicMock( + return_value=(True, "Successfully revealed weights.") + ) + self.subtensor._do_reveal_weights = MagicMock(return_value=(True, None)) + + # Commit weights with wait_for_inclusion + commit_success, commit_message = self.subtensor.commit_weights( + wallet=self.wallet, + netuid=1, + uids=uids, + weights=weights, + wait_for_inclusion=True, + ) + assert commit_success == True + assert commit_message == "Successfully committed weights." + + # Reveal weights with wait_for_inclusion + reveal_success, reveal_message = self.subtensor.reveal_weights( + wallet=self.wallet, + netuid=1, + uids=uids, + weights=weights, + version_key=version_key, + wait_for_inclusion=True, + ) + assert reveal_success == True + assert reveal_message == "Successfully revealed weights." + def test_get_balance(self): fake_coldkey = _get_mock_coldkey(0) balance = self.subtensor.get_balance(address=fake_coldkey) From 92b3b22cb4d6a6229b72b64b90a2cb48a658ffa3 Mon Sep 17 00:00:00 2001 From: opendansor Date: Fri, 24 May 2024 16:34:34 -0700 Subject: [PATCH 02/12] - remove torch - add e2e test - encode data using SCALE codec --- bittensor/chain_data.py | 6 + bittensor/cli.py | 12 +- bittensor/commands/network.py | 2 + bittensor/commands/weights.py | 61 ++++-- bittensor/subtensor.py | 56 ++++-- bittensor/utils/weight_utils.py | 31 ++- .../e2e_tests/subcommands/weights/__init__.py | 0 .../weights/test_commit_weights.py | 179 ++++++++++++++++++ .../integration_tests/test_cli_no_network.py | 2 +- .../test_subtensor_integration.py | 33 ++-- 10 files changed, 317 insertions(+), 65 deletions(-) create mode 100644 tests/e2e_tests/subcommands/weights/__init__.py create mode 100644 tests/e2e_tests/subcommands/weights/test_commit_weights.py diff --git a/bittensor/chain_data.py b/bittensor/chain_data.py index 548fa40ede..9140415506 100644 --- a/bittensor/chain_data.py +++ b/bittensor/chain_data.py @@ -177,6 +177,8 @@ ["max_validators", "Compact"], ["adjustment_alpha", "Compact"], ["difficulty", "Compact"], + ["commit_reveal_weights_interval", "Compact"], + ["commit_reveal_weights_enabled", "bool"], ], }, } @@ -1074,6 +1076,8 @@ class SubnetHyperparameters: max_validators: int adjustment_alpha: int difficulty: int + commit_reveal_weights_interval: int + commit_reveal_weights_enabled: bool @classmethod def from_vec_u8(cls, vec_u8: List[int]) -> Optional["SubnetHyperparameters"]: @@ -1128,6 +1132,8 @@ def fix_decoded_values(cls, decoded: Dict) -> "SubnetHyperparameters": bonds_moving_avg=decoded["bonds_moving_avg"], adjustment_alpha=decoded["adjustment_alpha"], difficulty=decoded["difficulty"], + commit_reveal_weights_interval=decoded["commit_reveal_weights_interval"], + commit_reveal_weights_enabled=decoded["commit_reveal_weights_enabled"], ) def _to_parameter_dict_torch( diff --git a/bittensor/cli.py b/bittensor/cli.py index 49e188317c..931c5d8d14 100644 --- a/bittensor/cli.py +++ b/bittensor/cli.py @@ -95,9 +95,9 @@ "sudos": "sudo", "i": "info", "info": "info", - "weights": "weight", - "wt": "weight", - "weight": "weight", + "weights": "weights", + "wt": "weights", + "weight": "weights", } COMMANDS = { "subnets": { @@ -171,9 +171,9 @@ "remove": UnStakeCommand, }, }, - "weight": { - "name": "weight", - "aliases": ["wt", "weights"], + "weights": { + "name": "weights", + "aliases": ["wt", "weight"], "help": "Commands for managing weight for subnets.", "commands": { "commit": CommitWeightCommand, diff --git a/bittensor/commands/network.py b/bittensor/commands/network.py index 64fbd272f6..f20eac67a6 100644 --- a/bittensor/commands/network.py +++ b/bittensor/commands/network.py @@ -323,6 +323,8 @@ def add_args(parser: argparse.ArgumentParser): "kappa": "sudo_set_kappa", "difficulty": "sudo_set_difficulty", "bonds_moving_avg": "sudo_set_bonds_moving_average", + "commit_reveal_weights_interval": "sudo_set_commit_reveal_weights_interval", + "commit_reveal_weights_enabled": "sudo_set_commit_reveal_weights_enabled", } diff --git a/bittensor/commands/weights.py b/bittensor/commands/weights.py index 9499c60c95..f62680a682 100644 --- a/bittensor/commands/weights.py +++ b/bittensor/commands/weights.py @@ -1,9 +1,27 @@ +# The MIT License (MIT) +# Copyright © 2021 Yuma Rao +# Copyright © 2023 Opentensor Foundation + +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the “Software”), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. + +# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + import argparse import re -import torch +import numpy as np from rich.prompt import Prompt - +import bittensor.utils.weight_utils as weight_utils import bittensor from . import defaults @@ -55,21 +73,22 @@ def _run(cli: "bittensor.cli", subtensor: "bittensor.subtensor"): # Parse from string netuid = cli.config.netuid - uids = torch.tensor( + uids = np.array( list(map(int, re.split(r"[ ,]+", cli.config.uids))), - dtype=torch.int64 + dtype=np.int64 ) - weights = torch.tensor( - list(map(int, re.split(r"[ ,]+", cli.config.weights))), - dtype=torch.float32 + weights = np.array( + list(map(float, re.split(r"[ ,]+", cli.config.weights))), + dtype=np.float32 ) + weight_uids, weight_vals = weight_utils.convert_weights_and_uids_for_emit(uids=uids, weights=weights) # Run the commit weights operation success, message = subtensor.commit_weights( wallet=wallet, netuid=netuid, - uids=uids, - weights=weights, + uids=weight_uids, + weights=weight_vals, wait_for_inclusion=cli.config.wait_for_inclusion, wait_for_finalization=cli.config.wait_for_finalization, prompt=cli.config.prompt, @@ -105,7 +124,7 @@ def add_args(parser: argparse.ArgumentParser): "--prompt", dest="prompt", action="store_true", - default=True, + default=False, ) bittensor.wallet.add_args(parser) @@ -140,9 +159,7 @@ class RevealWeightCommand: def run(cli: "bittensor.cli"): r"""Reveal weights for a specific subnet.""" try: - subtensor: "bittensor.subtensor" = bittensor.subtensor( - config=cli.config, log_verbose=False - ) + subtensor: "bittensor.subtensor" = bittensor.subtensor(config=cli.config, log_verbose=False) RevealWeightCommand._run(cli, subtensor) finally: if "subtensor" in locals(): @@ -166,22 +183,24 @@ def _run(cli: "bittensor.cli", subtensor: "bittensor.subtensor"): # Parse from string netuid = cli.config.netuid - uids = torch.tensor( + version = bittensor.__version_as_int__ + uids = np.array( list(map(int, re.split(r"[ ,]+", cli.config.uids))), - dtype=torch.int64, + dtype=np.int64, ) - weights = torch.tensor( + weights = np.array( list(map(float, re.split(r"[ ,]+", cli.config.weights))), - dtype=torch.float32, + dtype=np.float32, ) + weight_uids, weight_vals = weight_utils.convert_weights_and_uids_for_emit(uids=uids, weights=weights) # Run the reveal weights operation. success, message = subtensor.reveal_weights( wallet=wallet, netuid=netuid, - uids=uids, - weights=weights, - version_key=0, + uids=weight_uids, + weights=weight_vals, + version_key=version, wait_for_inclusion=cli.config.wait_for_inclusion, wait_for_finalization=cli.config.wait_for_finalization, prompt=cli.config.prompt, @@ -216,7 +235,7 @@ def add_args(parser: argparse.ArgumentParser): "--prompt", dest="prompt", action="store_true", - default=True, + default=False, ) bittensor.wallet.add_args(parser) diff --git a/bittensor/subtensor.py b/bittensor/subtensor.py index 2ed618fd5f..34d50bd9d0 100644 --- a/bittensor/subtensor.py +++ b/bittensor/subtensor.py @@ -910,8 +910,9 @@ def commit_weights( self, wallet: "bittensor.wallet", netuid: int, - uids: torch.Tensor, - weights: torch.Tensor, + uids: Union[NDArray[np.int64], list], + weights: Union[NDArray[np.float32], list], + version_key: int = bittensor.__version_as_int__, wait_for_inclusion: bool = False, wait_for_finalization: bool = False, prompt: bool = False, @@ -920,16 +921,22 @@ def commit_weights( """ Commits a hash of the neuron's weights to the Bittensor blockchain using the provided wallet. This action serves as a commitment or snapshot of the neuron's current weight distribution. + Args: wallet (bittensor.wallet): The wallet associated with the neuron committing the weights. netuid (int): The unique identifier of the subnet. + uids (np.ndarray): NumPy array of neuron UIDs for which weights are being committed. + weights (np.ndarray): NumPy array of weight values corresponding to each UID. + version_key (int, optional): Version key for compatibility with the network. wait_for_inclusion (bool, optional): Waits for the transaction to be included in a block. wait_for_finalization (bool, optional): Waits for the transaction to be finalized on the blockchain. prompt (bool, optional): If ``True``, prompts for user confirmation before proceeding. max_retries (int, optional): The number of maximum attempts to commit weights. (Default: 5) + Returns: Tuple[bool, str]: ``True`` if the weight commitment is successful, False otherwise. And `msg`, a string value describing the success or potential error. + This function allows neurons to create a tamper-proof record of their weight distribution at a specific point in time, enhancing transparency and accountability within the Bittensor network. """ @@ -938,15 +945,23 @@ def commit_weights( success = False message = "No attempt made. Perhaps it is too soon to commit weights!" + _logger.info( + "Committing weights with params: netuid={}, uids={}, weights={}, version_key={}".format( + netuid, uids, weights, version_key + ) + ) + # Generate the hash of the weights commit_hash = weight_utils.generate_weight_hash( who=wallet.hotkey.ss58_address, netuid=netuid, - uids=uids.tolist(), - values=weights.tolist(), - version_key=0, + uids=uids, + values=weights, + version_key=version_key, ) + _logger.info("Commit Hash: {}".format(commit_hash)) + while retries < max_retries: try: success, message = commit_weights_extrinsic( @@ -978,16 +993,19 @@ def _do_commit_weights( """ Internal method to send a transaction to the Bittensor blockchain, committing the hash of a neuron's weights. This method constructs and submits the transaction, handling retries and blockchain communication. + Args: wallet (bittensor.wallet): The wallet associated with the neuron committing the weights. netuid (int): The unique identifier of the subnet. commit_hash (str): The hash of the neuron's weights to be committed. wait_for_inclusion (bool, optional): Waits for the transaction to be included in a block. wait_for_finalization (bool, optional): Waits for the transaction to be finalized on the blockchain. + Returns: Tuple[bool, Optional[str]]: A tuple containing a success flag and an optional error message. - This method ensures that the weight commitment is securely recorded on the Bittensor blockchain, providing a verifiable - record of the neuron's weight distribution at a specific point in time. + + This method ensures that the weight commitment is securely recorded on the Bittensor blockchain, providing a + verifiable record of the neuron's weight distribution at a specific point in time. """ @retry(delay=1, tries=3, backoff=2, max_delay=4, logger=_logger) @@ -1028,9 +1046,9 @@ def reveal_weights( self, wallet: "bittensor.wallet", netuid: int, - uids: torch.Tensor, - weights: torch.Tensor, - version_key: int, + uids: Union[NDArray[np.int64], list], + weights: Union[NDArray[np.float32], list], + version_key: int = bittensor.__version_as_int__, wait_for_inclusion: bool = False, wait_for_finalization: bool = False, prompt: bool = False, @@ -1039,19 +1057,22 @@ def reveal_weights( """ Reveals the weights for a specific subnet on the Bittensor blockchain using the provided wallet. This action serves as a revelation of the neuron's previously committed weight distribution. + Args: wallet (bittensor.wallet): The wallet associated with the neuron revealing the weights. netuid (int): The unique identifier of the subnet. - uids (torch.Tensor): Tensor of neuron UIDs for which weights are being revealed. - weights (torch.Tensor): Tensor of weight values corresponding to each UID. - version_key (int): Version key for compatibility with the network. + uids (np.ndarray): NumPy array of neuron UIDs for which weights are being revealed. + weights (np.ndarray): NumPy array of weight values corresponding to each UID. + version_key (int, optional): Version key for compatibility with the network. wait_for_inclusion (bool, optional): Waits for the transaction to be included in a block. wait_for_finalization (bool, optional): Waits for the transaction to be finalized on the blockchain. prompt (bool, optional): If ``True``, prompts for user confirmation before proceeding. max_retries (int, optional): The number of maximum attempts to reveal weights. (Default: 5) + Returns: Tuple[bool, str]: ``True`` if the weight revelation is successful, False otherwise. And `msg`, a string value describing the success or potential error. + This function allows neurons to reveal their previously committed weight distribution, ensuring transparency and accountability within the Bittensor network. """ @@ -1066,8 +1087,8 @@ def reveal_weights( subtensor=self, wallet=wallet, netuid=netuid, - uids=uids.tolist(), - weights=weights.tolist(), + uids=uids, + weights=weights, version_key=version_key, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, @@ -1095,6 +1116,7 @@ def _do_reveal_weights( """ Internal method to send a transaction to the Bittensor blockchain, revealing the weights for a specific subnet. This method constructs and submits the transaction, handling retries and blockchain communication. + Args: wallet (bittensor.wallet): The wallet associated with the neuron revealing the weights. netuid (int): The unique identifier of the subnet. @@ -1103,13 +1125,15 @@ def _do_reveal_weights( version_key (int): Version key for compatibility with the network. wait_for_inclusion (bool, optional): Waits for the transaction to be included in a block. wait_for_finalization (bool, optional): Waits for the transaction to be finalized on the blockchain. + Returns: Tuple[bool, Optional[str]]: A tuple containing a success flag and an optional error message. + This method ensures that the weight revelation is securely recorded on the Bittensor blockchain, providing transparency and accountability for the neuron's weight distribution. """ - @retry(delay=2, tries=3, backoff=2, max_delay=4, logger=_logger) + @retry(delay=1, tries=3, backoff=2, max_delay=4, logger=_logger) def make_substrate_call_with_retry(): call = self.substrate.compose_call( call_module="SubtensorModule", diff --git a/bittensor/utils/weight_utils.py b/bittensor/utils/weight_utils.py index c78095008d..94262cafc9 100644 --- a/bittensor/utils/weight_utils.py +++ b/bittensor/utils/weight_utils.py @@ -1,4 +1,5 @@ -""" Conversion for weight between chain representation and np.array or torch.Tensor +""" +Conversion for weight between chain representation and np.array or torch.Tensor """ # The MIT License (MIT) @@ -21,6 +22,9 @@ import numpy as np import bittensor import hashlib +import struct +from scalecodec import ScaleBytes, U16, Vec +from substrateinterface import Keypair from numpy.typing import NDArray from typing import Tuple, List, Union from bittensor.utils.registration import torch, use_torch, legacy_torch_api_compat @@ -349,22 +353,39 @@ def generate_weight_hash( ) -> str: """ Generate a valid commit hash from the provided weights. + Args: who (str): The account identifier. netuid (int): The network unique identifier. uids (List[int]): The list of UIDs. values (List[int]): The list of weight values. version_key (int): The version key. + Returns: str: The generated commit hash. """ - # Create a tuple of the input parameters - data = (who, netuid, uids, values, version_key) + # Encode data using SCALE codec + the_who = ScaleBytes(Keypair(ss58_address=who).public_key) + the_netuid = ScaleBytes(netuid.to_bytes(2, "little")) + + vec_uids = Vec(data=None, sub_type="U16") + vec_uids.value = [U16(ScaleBytes(uid.to_bytes(2, "little"))) for uid in uids] + the_uids = ScaleBytes(vec_uids.encode().data) + + vec_values = Vec(data=None, sub_type="U16") + vec_values.value = [ + U16(ScaleBytes(bytearray(struct.pack(" 0, f"Invalid block number: {commit_block}" + + # Query the WeightCommitRevealInterval storage map + weight_commit_reveal_interval = subtensor.query_module( + module="SubtensorModule", name="WeightCommitRevealInterval", params=[1] + ) + interval = weight_commit_reveal_interval.value + assert interval > 0, "Invalid WeightCommitRevealInterval" + + # Wait until the reveal block range + current_block = subtensor.get_current_block() + reveal_block_start = (commit_block - (commit_block % interval)) + interval + while current_block < reveal_block_start: + time.sleep(1) # Wait for 1 second before checking the block number again + current_block = subtensor.get_current_block() + if current_block % 10 == 0: + print(f"Current Block: {current_block} Revealing at: {reveal_block_start}") + + # Configure the CLI arguments for the RevealWeightCommand + exec_command( + RevealWeightCommand, + [ + "wt", + "reveal", + "--no_prompt", + "--netuid", + "1", + "--uids", + str(uid), + "--weights", + str(weights), + "--subtensor.network", + "local", + "--subtensor.chain_endpoint", + "ws://localhost:9945", + "--wallet.path", + "/tmp/btcli-wallet", + ], + ) + + # Query the Weights storage map + revealed_weights = subtensor.query_module( + module="SubtensorModule", name="Weights", params=[1, uid] # netuid and uid + ) + + # Assert that the revealed weights are set correctly + assert revealed_weights.value is not None, "Weight reveal not found in storage" + + uid_list = list(map(int, re.split(r"[ ,]+", str(uid)))) + uids = np.array(uid_list, dtype=np.int64) + weight_list = list(map(float, re.split(r"[ ,]+", str(weights)))) + weights_array = np.array(weight_list, dtype=np.float32) + weight_uids, expected_weights = weight_utils.convert_weights_and_uids_for_emit( + uids, weights_array + ) + assert ( + expected_weights[0] == revealed_weights.value[0][1] + ), f"Incorrect revealed weights. Expected: {expected_weights[0]}, Actual: {revealed_weights.value[0][1]}" diff --git a/tests/integration_tests/test_cli_no_network.py b/tests/integration_tests/test_cli_no_network.py index 6d6d24f58a..d2bc78f3aa 100644 --- a/tests/integration_tests/test_cli_no_network.py +++ b/tests/integration_tests/test_cli_no_network.py @@ -427,7 +427,7 @@ def test_command_no_args(self, _, __, patched_prompt_ask): "stakes", "roots", "wallets", - "weights", + "weight", "st", "wt", "su", diff --git a/tests/integration_tests/test_subtensor_integration.py b/tests/integration_tests/test_subtensor_integration.py index ea9f2d2822..1ff50be3bf 100644 --- a/tests/integration_tests/test_subtensor_integration.py +++ b/tests/integration_tests/test_subtensor_integration.py @@ -24,6 +24,7 @@ import pytest import torch from substrateinterface import Keypair +import numpy as np import bittensor from bittensor.mock import MockSubtensor @@ -361,8 +362,8 @@ def test_set_weights_failed(self): assert fail == False def test_commit_weights(self): - weights = torch.FloatTensor([0.1, 0.2, 0.3, 0.4]) - uids = torch.tensor([1, 2, 3, 4], dtype=torch.int64) + weights = np.array([0.1, 0.2, 0.3, 0.4], dtype=np.float32) + uids = np.array([1, 2, 3, 4], dtype=np.int64) commit_hash = bittensor.utils.weight_utils.generate_weight_hash( who=self.wallet.hotkey.ss58_address, netuid=3, @@ -386,8 +387,8 @@ def test_commit_weights(self): assert message == "Successfully committed weights." def test_commit_weights_inclusion(self): - weights = torch.FloatTensor([0.1, 0.2, 0.3, 0.4]) - uids = torch.tensor([1, 2, 3, 4], dtype=torch.int64) + weights = np.array([0.1, 0.2, 0.3, 0.4], dtype=np.float32) + uids = np.array([1, 2, 3, 4], dtype=np.int64) commit_hash = bittensor.utils.weight_utils.generate_weight_hash( who=self.wallet.hotkey.ss58_address, netuid=1, @@ -412,8 +413,8 @@ def test_commit_weights_inclusion(self): assert message == "Successfully committed weights." def test_commit_weights_failed(self): - weights = torch.FloatTensor([0.1, 0.2, 0.3, 0.4]) - uids = torch.tensor([1, 2, 3, 4], dtype=torch.int64) + weights = np.array([0.1, 0.2, 0.3, 0.4], dtype=np.float32) + uids = np.array([1, 2, 3, 4], dtype=np.int64) commit_hash = bittensor.utils.weight_utils.generate_weight_hash( who=self.wallet.hotkey.ss58_address, netuid=3, @@ -440,8 +441,8 @@ def test_commit_weights_failed(self): assert message == "Mock failure message" def test_reveal_weights(self): - weights = torch.FloatTensor([0.1, 0.2, 0.3, 0.4]) - uids = torch.tensor([1, 2, 3, 4], dtype=torch.int64) + weights = np.array([0.1, 0.2, 0.3, 0.4], dtype=np.float32) + uids = np.array([1, 2, 3, 4], dtype=np.int64) self.subtensor.reveal_weights = MagicMock( return_value=(True, "Successfully revealed weights.") @@ -455,8 +456,8 @@ def test_reveal_weights(self): assert message == "Successfully revealed weights." def test_reveal_weights_inclusion(self): - weights = torch.FloatTensor([0.1, 0.2, 0.3, 0.4]) - uids = torch.tensor([1, 2, 3, 4], dtype=torch.int64) + weights = np.array([0.1, 0.2, 0.3, 0.4], dtype=np.float32) + uids = np.array([1, 2, 3, 4], dtype=np.int64) self.subtensor._do_reveal_weights = MagicMock(return_value=(True, None)) self.subtensor.reveal_weights = MagicMock( @@ -475,8 +476,8 @@ def test_reveal_weights_inclusion(self): assert message == "Successfully revealed weights." def test_reveal_weights_failed(self): - weights = torch.FloatTensor([0.1, 0.2, 0.3, 0.4]) - uids = torch.tensor([1, 2, 3, 4], dtype=torch.int64) + weights = np.array([0.1, 0.2, 0.3, 0.4], dtype=np.float32) + uids = np.array([1, 2, 3, 4], dtype=np.int64) self.subtensor._do_reveal_weights = MagicMock( return_value=(False, "Mock failure message") @@ -497,8 +498,8 @@ def test_reveal_weights_failed(self): assert message == "Mock failure message" def test_commit_and_reveal_weights(self): - weights = torch.FloatTensor([0.1, 0.2, 0.3, 0.4]) - uids = torch.tensor([1, 2, 3, 4], dtype=torch.int64) + weights = np.array([0.1, 0.2, 0.3, 0.4], dtype=np.float32) + uids = np.array([1, 2, 3, 4], dtype=np.int64) version_key = 0 # Mock the commit_weights and reveal_weights functions @@ -533,8 +534,8 @@ def test_commit_and_reveal_weights(self): assert reveal_message == "Successfully revealed weights." def test_commit_and_reveal_weights_inclusion(self): - weights = torch.FloatTensor([0.1, 0.2, 0.3, 0.4]) - uids = torch.tensor([1, 2, 3, 4], dtype=torch.int64) + weights = np.array([0.1, 0.2, 0.3, 0.4], dtype=np.float32) + uids = np.array([1, 2, 3, 4], dtype=np.int64) version_key = 0 # Mock the commit_weights and reveal_weights functions From ad0cf0353766e9e18da8f8190975a8e5c11af083 Mon Sep 17 00:00:00 2001 From: opendansor Date: Fri, 24 May 2024 16:38:19 -0700 Subject: [PATCH 03/12] black reformat --- bittensor/commands/weights.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/bittensor/commands/weights.py b/bittensor/commands/weights.py index f62680a682..510ef3d414 100644 --- a/bittensor/commands/weights.py +++ b/bittensor/commands/weights.py @@ -49,7 +49,9 @@ class CommitWeightCommand: def run(cli: "bittensor.cli"): r"""Commit weights for a specific subnet.""" try: - subtensor: "bittensor.subtensor" = bittensor.subtensor(config=cli.config, log_verbose=False) + subtensor: "bittensor.subtensor" = bittensor.subtensor( + config=cli.config, log_verbose=False + ) CommitWeightCommand._run(cli, subtensor) finally: if "subtensor" in locals(): @@ -74,14 +76,14 @@ def _run(cli: "bittensor.cli", subtensor: "bittensor.subtensor"): # Parse from string netuid = cli.config.netuid uids = np.array( - list(map(int, re.split(r"[ ,]+", cli.config.uids))), - dtype=np.int64 + list(map(int, re.split(r"[ ,]+", cli.config.uids))), dtype=np.int64 ) weights = np.array( - list(map(float, re.split(r"[ ,]+", cli.config.weights))), - dtype=np.float32 + list(map(float, re.split(r"[ ,]+", cli.config.weights))), dtype=np.float32 + ) + weight_uids, weight_vals = weight_utils.convert_weights_and_uids_for_emit( + uids=uids, weights=weights ) - weight_uids, weight_vals = weight_utils.convert_weights_and_uids_for_emit(uids=uids, weights=weights) # Run the commit weights operation success, message = subtensor.commit_weights( @@ -159,7 +161,9 @@ class RevealWeightCommand: def run(cli: "bittensor.cli"): r"""Reveal weights for a specific subnet.""" try: - subtensor: "bittensor.subtensor" = bittensor.subtensor(config=cli.config, log_verbose=False) + subtensor: "bittensor.subtensor" = bittensor.subtensor( + config=cli.config, log_verbose=False + ) RevealWeightCommand._run(cli, subtensor) finally: if "subtensor" in locals(): @@ -192,7 +196,9 @@ def _run(cli: "bittensor.cli", subtensor: "bittensor.subtensor"): list(map(float, re.split(r"[ ,]+", cli.config.weights))), dtype=np.float32, ) - weight_uids, weight_vals = weight_utils.convert_weights_and_uids_for_emit(uids=uids, weights=weights) + weight_uids, weight_vals = weight_utils.convert_weights_and_uids_for_emit( + uids=uids, weights=weights + ) # Run the reveal weights operation. success, message = subtensor.reveal_weights( From 84ac53fcb6d3de4f8afd574d0c134fc2b5bffb10 Mon Sep 17 00:00:00 2001 From: opendansor Date: Fri, 24 May 2024 17:04:00 -0700 Subject: [PATCH 04/12] mypy reformat --- bittensor/subtensor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bittensor/subtensor.py b/bittensor/subtensor.py index c29c6e95bf..5a297d938d 100644 --- a/bittensor/subtensor.py +++ b/bittensor/subtensor.py @@ -955,8 +955,8 @@ def commit_weights( commit_hash = weight_utils.generate_weight_hash( who=wallet.hotkey.ss58_address, netuid=netuid, - uids=uids, - values=weights, + uids=list(uids), + values=list(weights), version_key=version_key, ) @@ -1087,8 +1087,8 @@ def reveal_weights( subtensor=self, wallet=wallet, netuid=netuid, - uids=uids, - weights=weights, + uids=list(uids), + weights=list(weights), version_key=version_key, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, From a8ced82d5233cda7323dde8d123f9bfcd38c3d86 Mon Sep 17 00:00:00 2001 From: opendansor Date: Tue, 28 May 2024 12:34:24 -0700 Subject: [PATCH 05/12] Add a salt function for hashing, address PR comments. --- bittensor/commands/weights.py | 40 ++++++++++--- bittensor/extrinsics/commit_weights.py | 11 +++- bittensor/subtensor.py | 12 +++- bittensor/utils/weight_utils.py | 29 +++++++--- .../weights/test_commit_weights.py | 4 +- .../test_subtensor_integration.py | 58 +++++++++++++------ 6 files changed, 112 insertions(+), 42 deletions(-) diff --git a/bittensor/commands/weights.py b/bittensor/commands/weights.py index 510ef3d414..a8da82e171 100644 --- a/bittensor/commands/weights.py +++ b/bittensor/commands/weights.py @@ -17,14 +17,18 @@ # DEALINGS IN THE SOFTWARE. import argparse +import os import re import numpy as np -from rich.prompt import Prompt +from rich.prompt import Prompt, Confirm import bittensor.utils.weight_utils as weight_utils import bittensor from . import defaults +"""Module that encapsulates the CommitWeightCommand and the RevealWeightCommand. Used to commit and reveal weights +for a specific subnet on the Bittensor Network.""" + class CommitWeightCommand: """ @@ -76,21 +80,30 @@ def _run(cli: "bittensor.cli", subtensor: "bittensor.subtensor"): # Parse from string netuid = cli.config.netuid uids = np.array( - list(map(int, re.split(r"[ ,]+", cli.config.uids))), dtype=np.int64 + [int(x) for x in re.split(r"[ ,]+", cli.config.uids)], dtype=np.int64 ) weights = np.array( - list(map(float, re.split(r"[ ,]+", cli.config.weights))), dtype=np.float32 + [float(x) for x in re.split(r"[ ,]+", cli.config.weights)], dtype=np.float32 ) weight_uids, weight_vals = weight_utils.convert_weights_and_uids_for_emit( uids=uids, weights=weights ) + # Generate random salt + salt_length = 8 + salt = list(os.urandom(salt_length)) + + if not Confirm.ask(f"Have you recorded the [red]salt[/red]: [bold white]'{salt}'[/bold white]? It will be " + f"required to reveal weights."): + return False, "User cancelled the operation." + # Run the commit weights operation success, message = subtensor.commit_weights( wallet=wallet, netuid=netuid, uids=weight_uids, weights=weight_vals, + salt=salt, wait_for_inclusion=cli.config.wait_for_inclusion, wait_for_finalization=cli.config.wait_for_finalization, prompt=cli.config.prompt, @@ -110,6 +123,7 @@ def add_args(parser: argparse.ArgumentParser): parser.add_argument("--netuid", dest="netuid", type=int, required=False) parser.add_argument("--uids", dest="uids", type=str, required=False) parser.add_argument("--weights", dest="weights", type=str, required=False) + parser.add_argument("--salt", dest="salt", type=str, required=False) parser.add_argument( "--wait-for-inclusion", dest="wait_for_inclusion", @@ -134,10 +148,10 @@ def add_args(parser: argparse.ArgumentParser): @staticmethod def check_config(config: "bittensor.config"): - if not config.is_set("wallet.name") and not config.no_prompt: + if not config.no_prompt and not config.is_set("wallet.name"): wallet_name = Prompt.ask("Enter wallet name", default=defaults.wallet.name) config.wallet.name = str(wallet_name) - if not config.is_set("wallet.hotkey") and not config.no_prompt: + if not config.no_prompt and not config.is_set("wallet.hotkey"): hotkey = Prompt.ask("Enter hotkey name", default=defaults.wallet.hotkey) config.wallet.hotkey = str(hotkey) @@ -151,8 +165,9 @@ class RevealWeightCommand: - ``--netuid`` (int): The netuid of the subnet for which weights are to be revealed. - ``--uids`` (str): Corresponding UIDs for the specified netuid, in comma-separated format. - ``--weights`` (str): Corresponding weights for the specified UIDs, in comma-separated format. + - ``--salt`` (str): Corresponding salt for the hash function, integers in comma-separated format. Example usage:: - $ btcli wt reveal --netuid 1 --uids 1,2,3,4 --weights 0.1,0.2,0.3,0.4 + $ btcli wt reveal --netuid 1 --uids 1,2,3,4 --weights 0.1,0.2,0.3,0.4 --salt 163,241,217,11,161,142,147,189 Note: This command is used to reveal weights for a specific subnet and requires the user to have the necessary permissions. """ @@ -185,17 +200,24 @@ def _run(cli: "bittensor.cli", subtensor: "bittensor.subtensor"): if not cli.config.is_set("weights"): cli.config.weights = Prompt.ask(f"Enter weights (comma-separated)") + if not cli.config.is_set("salt"): + cli.config.salt = Prompt.ask(f"Enter salt (comma-separated)") + # Parse from string netuid = cli.config.netuid version = bittensor.__version_as_int__ uids = np.array( - list(map(int, re.split(r"[ ,]+", cli.config.uids))), + [int(x) for x in re.split(r"[ ,]+", cli.config.uids)], dtype=np.int64, ) weights = np.array( - list(map(float, re.split(r"[ ,]+", cli.config.weights))), + [float(x) for x in re.split(r"[ ,]+", cli.config.weights)], dtype=np.float32, ) + salt = np.array( + [int(x) for x in re.split(r"[ ,]+", cli.config.salt)], + dtype=np.int64, + ) weight_uids, weight_vals = weight_utils.convert_weights_and_uids_for_emit( uids=uids, weights=weights ) @@ -206,6 +228,7 @@ def _run(cli: "bittensor.cli", subtensor: "bittensor.subtensor"): netuid=netuid, uids=weight_uids, weights=weight_vals, + salt=salt, version_key=version, wait_for_inclusion=cli.config.wait_for_inclusion, wait_for_finalization=cli.config.wait_for_finalization, @@ -225,6 +248,7 @@ def add_args(parser: argparse.ArgumentParser): parser.add_argument("--netuid", dest="netuid", type=int, required=False) parser.add_argument("--uids", dest="uids", type=str, required=False) parser.add_argument("--weights", dest="weights", type=str, required=False) + parser.add_argument("--salt", dest="salt", type=str, required=False) parser.add_argument( "--wait-for-inclusion", dest="wait_for_inclusion", diff --git a/bittensor/extrinsics/commit_weights.py b/bittensor/extrinsics/commit_weights.py index 59fd2f2776..669a4bdcc1 100644 --- a/bittensor/extrinsics/commit_weights.py +++ b/bittensor/extrinsics/commit_weights.py @@ -17,9 +17,12 @@ # DEALINGS IN THE SOFTWARE. from typing import Tuple, List +from rich.prompt import Confirm import bittensor +""" Module commit weights and reveal weights extrinsic. """ + def commit_weights_extrinsic( subtensor: "bittensor.subtensor", @@ -47,9 +50,8 @@ def commit_weights_extrinsic( This function provides a user-friendly interface for committing weights to the Bittensor blockchain, ensuring proper error handling and user interaction when required. """ - if prompt: - if not input("Would you like to commit weights? (y/n) ").lower() == "y": - return False, "User cancelled the operation." + if prompt and not Confirm.ask(f"Would you like to commit weights?"): + return False, "User cancelled the operation." success, error_message = subtensor._do_commit_weights( wallet=wallet, @@ -73,6 +75,7 @@ def reveal_weights_extrinsic( netuid: int, uids: List[int], weights: List[int], + salt: List[int], version_key: int, wait_for_inclusion: bool = False, wait_for_finalization: bool = False, @@ -87,6 +90,7 @@ def reveal_weights_extrinsic( netuid (int): The unique identifier of the subnet. uids (List[int]): List of neuron UIDs for which weights are being revealed. weights (List[int]): List of weight values corresponding to each UID. + salt (List[int]): List of salt values corresponding to the hash function. version_key (int): Version key for compatibility with the network. wait_for_inclusion (bool, optional): Waits for the transaction to be included in a block. wait_for_finalization (bool, optional): Waits for the transaction to be finalized on the blockchain. @@ -106,6 +110,7 @@ def reveal_weights_extrinsic( netuid=netuid, uids=uids, values=weights, + salt=salt, version_key=version_key, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, diff --git a/bittensor/subtensor.py b/bittensor/subtensor.py index 5a297d938d..3d8ea92107 100644 --- a/bittensor/subtensor.py +++ b/bittensor/subtensor.py @@ -23,6 +23,7 @@ import argparse import copy import functools +import os import socket import time from typing import List, Dict, Union, Optional, Tuple, TypedDict, Any @@ -910,6 +911,7 @@ def commit_weights( self, wallet: "bittensor.wallet", netuid: int, + salt: List[int], uids: Union[NDArray[np.int64], list], weights: Union[NDArray[np.float32], list], version_key: int = bittensor.__version_as_int__, @@ -925,6 +927,7 @@ def commit_weights( Args: wallet (bittensor.wallet): The wallet associated with the neuron committing the weights. netuid (int): The unique identifier of the subnet. + salt (List[int]): list of randomly generated integers as salt to generated weighted hash. uids (np.ndarray): NumPy array of neuron UIDs for which weights are being committed. weights (np.ndarray): NumPy array of weight values corresponding to each UID. version_key (int, optional): Version key for compatibility with the network. @@ -953,10 +956,11 @@ def commit_weights( # Generate the hash of the weights commit_hash = weight_utils.generate_weight_hash( - who=wallet.hotkey.ss58_address, + address=wallet.hotkey.ss58_address, netuid=netuid, uids=list(uids), values=list(weights), + salt=salt, version_key=version_key, ) @@ -1048,6 +1052,7 @@ def reveal_weights( netuid: int, uids: Union[NDArray[np.int64], list], weights: Union[NDArray[np.float32], list], + salt: Union[NDArray[np.int64], list], version_key: int = bittensor.__version_as_int__, wait_for_inclusion: bool = False, wait_for_finalization: bool = False, @@ -1063,6 +1068,7 @@ def reveal_weights( netuid (int): The unique identifier of the subnet. uids (np.ndarray): NumPy array of neuron UIDs for which weights are being revealed. weights (np.ndarray): NumPy array of weight values corresponding to each UID. + salt (np.ndarray): NumPy array of salt values corresponding to the hash function. version_key (int, optional): Version key for compatibility with the network. wait_for_inclusion (bool, optional): Waits for the transaction to be included in a block. wait_for_finalization (bool, optional): Waits for the transaction to be finalized on the blockchain. @@ -1089,6 +1095,7 @@ def reveal_weights( netuid=netuid, uids=list(uids), weights=list(weights), + salt=list(salt), version_key=version_key, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, @@ -1109,6 +1116,7 @@ def _do_reveal_weights( netuid: int, uids: List[int], values: List[int], + salt: List[int], version_key: int, wait_for_inclusion: bool = False, wait_for_finalization: bool = False, @@ -1122,6 +1130,7 @@ def _do_reveal_weights( netuid (int): The unique identifier of the subnet. uids (List[int]): List of neuron UIDs for which weights are being revealed. values (List[int]): List of weight values corresponding to each UID. + salt (List[int]): List of salt values corresponding to the hash function. version_key (int): Version key for compatibility with the network. wait_for_inclusion (bool, optional): Waits for the transaction to be included in a block. wait_for_finalization (bool, optional): Waits for the transaction to be finalized on the blockchain. @@ -1142,6 +1151,7 @@ def make_substrate_call_with_retry(): "netuid": netuid, "uids": uids, "values": values, + "salt": salt, "version_key": version_key, }, ) diff --git a/bittensor/utils/weight_utils.py b/bittensor/utils/weight_utils.py index 4f4f513d82..f624aa8a96 100644 --- a/bittensor/utils/weight_utils.py +++ b/bittensor/utils/weight_utils.py @@ -1,6 +1,7 @@ """ Conversion for weight between chain representation and np.array or torch.Tensor """ +import os # The MIT License (MIT) # Copyright © 2021 Yuma Rao @@ -349,15 +350,21 @@ def process_weights_for_netuid( def generate_weight_hash( - who: str, netuid: int, uids: List[int], values: List[int], version_key: int + address: str, + netuid: int, + uids: List[int], + values: List[int], + version_key: int, + salt: List[int], ) -> str: """ Generate a valid commit hash from the provided weights. Args: - who (str): The account identifier. + address (str): The account identifier. Wallet ss58_address. netuid (int): The network unique identifier. uids (List[int]): The list of UIDs. + salt (List[int]): The salt to add to hash. values (List[int]): The list of weight values. version_key (int): The version key. @@ -365,25 +372,29 @@ def generate_weight_hash( str: The generated commit hash. """ # Encode data using SCALE codec - the_who = ScaleBytes(Keypair(ss58_address=who).public_key) - the_netuid = ScaleBytes(netuid.to_bytes(2, "little")) + wallet_address = ScaleBytes(Keypair(ss58_address=address).public_key) + netuid = ScaleBytes(netuid.to_bytes(2, "little")) vec_uids = Vec(data=None, sub_type="U16") vec_uids.value = [U16(ScaleBytes(uid.to_bytes(2, "little"))) for uid in uids] - the_uids = ScaleBytes(vec_uids.encode().data) + uids = ScaleBytes(vec_uids.encode().data) vec_values = Vec(data=None, sub_type="U16") vec_values.value = [ U16(ScaleBytes(bytearray(struct.pack(" Date: Tue, 28 May 2024 12:49:23 -0700 Subject: [PATCH 06/12] Replace prompt with Confirm.ask, run black. --- bittensor/commands/weights.py | 6 ++++-- bittensor/extrinsics/commit_weights.py | 6 +++--- tests/integration_tests/test_subtensor_integration.py | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/bittensor/commands/weights.py b/bittensor/commands/weights.py index a8da82e171..16f50eb726 100644 --- a/bittensor/commands/weights.py +++ b/bittensor/commands/weights.py @@ -93,8 +93,10 @@ def _run(cli: "bittensor.cli", subtensor: "bittensor.subtensor"): salt_length = 8 salt = list(os.urandom(salt_length)) - if not Confirm.ask(f"Have you recorded the [red]salt[/red]: [bold white]'{salt}'[/bold white]? It will be " - f"required to reveal weights."): + if not Confirm.ask( + f"Have you recorded the [red]salt[/red]: [bold white]'{salt}'[/bold white]? It will be " + f"required to reveal weights." + ): return False, "User cancelled the operation." # Run the commit weights operation diff --git a/bittensor/extrinsics/commit_weights.py b/bittensor/extrinsics/commit_weights.py index 669a4bdcc1..ee1b0ef0d7 100644 --- a/bittensor/extrinsics/commit_weights.py +++ b/bittensor/extrinsics/commit_weights.py @@ -101,9 +101,9 @@ def reveal_weights_extrinsic( This function provides a user-friendly interface for revealing weights on the Bittensor blockchain, ensuring proper error handling and user interaction when required. """ - if prompt: - if not input("Would you like to reveal weights? (y/n) ").lower() == "y": - return False, "User cancelled the operation." + + if prompt and not Confirm.ask(f"Would you like to reveal weights?"): + return False, "User cancelled the operation." success, error_message = subtensor._do_reveal_weights( wallet=wallet, diff --git a/tests/integration_tests/test_subtensor_integration.py b/tests/integration_tests/test_subtensor_integration.py index 672f8437e6..311f58687b 100644 --- a/tests/integration_tests/test_subtensor_integration.py +++ b/tests/integration_tests/test_subtensor_integration.py @@ -373,7 +373,7 @@ def test_commit_weights(self): version_key=0, ) - self.subtensor.commit_weights = MagicMock( + self.subtensor.commit_weights = mocker.MagicMock( return_value=(True, "Successfully committed weights.") ) self.subtensor._do_commit_weights = MagicMock(return_value=(True, None)) From 4fe438b0b86214ed447710267193e29f44e1645a Mon Sep 17 00:00:00 2001 From: opendansor Date: Tue, 28 May 2024 13:02:17 -0700 Subject: [PATCH 07/12] Remove mocker. --- tests/integration_tests/test_subtensor_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_tests/test_subtensor_integration.py b/tests/integration_tests/test_subtensor_integration.py index 311f58687b..672f8437e6 100644 --- a/tests/integration_tests/test_subtensor_integration.py +++ b/tests/integration_tests/test_subtensor_integration.py @@ -373,7 +373,7 @@ def test_commit_weights(self): version_key=0, ) - self.subtensor.commit_weights = mocker.MagicMock( + self.subtensor.commit_weights = MagicMock( return_value=(True, "Successfully committed weights.") ) self.subtensor._do_commit_weights = MagicMock(return_value=(True, None)) From d7be84373b1eb531892a3c1575bbe9f74334dd6b Mon Sep 17 00:00:00 2001 From: opendansor Date: Tue, 28 May 2024 13:20:15 -0700 Subject: [PATCH 08/12] Flake8 --- bittensor/subtensor.py | 1 - bittensor/utils/weight_utils.py | 1 - 2 files changed, 2 deletions(-) diff --git a/bittensor/subtensor.py b/bittensor/subtensor.py index 3d8ea92107..49ed50b6f1 100644 --- a/bittensor/subtensor.py +++ b/bittensor/subtensor.py @@ -23,7 +23,6 @@ import argparse import copy import functools -import os import socket import time from typing import List, Dict, Union, Optional, Tuple, TypedDict, Any diff --git a/bittensor/utils/weight_utils.py b/bittensor/utils/weight_utils.py index f624aa8a96..d8f6dd0035 100644 --- a/bittensor/utils/weight_utils.py +++ b/bittensor/utils/weight_utils.py @@ -1,7 +1,6 @@ """ Conversion for weight between chain representation and np.array or torch.Tensor """ -import os # The MIT License (MIT) # Copyright © 2021 Yuma Rao From 668fd29bd505a1c6fc3f78dc6f17c5d45efffcb0 Mon Sep 17 00:00:00 2001 From: opendansor Date: Tue, 28 May 2024 14:23:41 -0700 Subject: [PATCH 09/12] E2E Test --- bittensor/subtensor.py | 12 ++++++------ bittensor/utils/weight_utils.py | 13 +++++++------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/bittensor/subtensor.py b/bittensor/subtensor.py index 49ed50b6f1..5cc1d5da06 100644 --- a/bittensor/subtensor.py +++ b/bittensor/subtensor.py @@ -56,6 +56,10 @@ custom_rpc_type_registry, ) from .errors import IdentityError, NominationError, StakeError, TakeError +from .extrinsics.commit_weights import ( + commit_weights_extrinsic, + reveal_weights_extrinsic, +) from .extrinsics.delegation import ( delegate_extrinsic, nominate_extrinsic, @@ -86,10 +90,6 @@ publish_metadata, get_metadata, ) -from .extrinsics.commit_weights import ( - commit_weights_extrinsic, - reveal_weights_extrinsic, -) from .extrinsics.set_weights import set_weights_extrinsic from .extrinsics.staking import add_stake_extrinsic, add_stake_multiple_extrinsic from .extrinsics.transfer import transfer_extrinsic @@ -103,8 +103,8 @@ ) from .utils.balance import Balance from .utils.registration import POWSolution -from .utils.subtensor import get_subtensor_errors from .utils.registration import legacy_torch_api_compat +from .utils.subtensor import get_subtensor_errors KEY_NONCE: Dict[str, int] = {} @@ -912,7 +912,7 @@ def commit_weights( netuid: int, salt: List[int], uids: Union[NDArray[np.int64], list], - weights: Union[NDArray[np.float32], list], + weights: Union[NDArray[np.int64], list], version_key: int = bittensor.__version_as_int__, wait_for_inclusion: bool = False, wait_for_finalization: bool = False, diff --git a/bittensor/utils/weight_utils.py b/bittensor/utils/weight_utils.py index d8f6dd0035..c7e25cf6a5 100644 --- a/bittensor/utils/weight_utils.py +++ b/bittensor/utils/weight_utils.py @@ -19,14 +19,15 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. -import numpy as np -import bittensor import hashlib -import struct +from typing import Tuple, List, Union + +import numpy as np +from numpy.typing import NDArray from scalecodec import ScaleBytes, U16, Vec from substrateinterface import Keypair -from numpy.typing import NDArray -from typing import Tuple, List, Union + +import bittensor from bittensor.utils.registration import torch, use_torch, legacy_torch_api_compat U32_MAX = 4294967295 @@ -380,7 +381,7 @@ def generate_weight_hash( vec_values = Vec(data=None, sub_type="U16") vec_values.value = [ - U16(ScaleBytes(bytearray(struct.pack(" Date: Tue, 28 May 2024 14:29:09 -0700 Subject: [PATCH 10/12] Float -> int for np --- bittensor/commands/weights.py | 9 +++++---- bittensor/extrinsics/commit_weights.py | 5 +++-- bittensor/subtensor.py | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/bittensor/commands/weights.py b/bittensor/commands/weights.py index 16f50eb726..9df3241bc0 100644 --- a/bittensor/commands/weights.py +++ b/bittensor/commands/weights.py @@ -16,6 +16,10 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. +"""Module that encapsulates the CommitWeightCommand and the RevealWeightCommand. Used to commit and reveal weights +for a specific subnet on the Bittensor Network.""" + + import argparse import os import re @@ -24,10 +28,7 @@ from rich.prompt import Prompt, Confirm import bittensor.utils.weight_utils as weight_utils import bittensor -from . import defaults - -"""Module that encapsulates the CommitWeightCommand and the RevealWeightCommand. Used to commit and reveal weights -for a specific subnet on the Bittensor Network.""" +from . import defaults # type: ignore class CommitWeightCommand: diff --git a/bittensor/extrinsics/commit_weights.py b/bittensor/extrinsics/commit_weights.py index ee1b0ef0d7..a27e1941ba 100644 --- a/bittensor/extrinsics/commit_weights.py +++ b/bittensor/extrinsics/commit_weights.py @@ -16,13 +16,14 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. +""" Module commit weights and reveal weights extrinsic. """ + from typing import Tuple, List + from rich.prompt import Confirm import bittensor -""" Module commit weights and reveal weights extrinsic. """ - def commit_weights_extrinsic( subtensor: "bittensor.subtensor", diff --git a/bittensor/subtensor.py b/bittensor/subtensor.py index 5cc1d5da06..ff48a50288 100644 --- a/bittensor/subtensor.py +++ b/bittensor/subtensor.py @@ -1050,7 +1050,7 @@ def reveal_weights( wallet: "bittensor.wallet", netuid: int, uids: Union[NDArray[np.int64], list], - weights: Union[NDArray[np.float32], list], + weights: Union[NDArray[np.int64], list], salt: Union[NDArray[np.int64], list], version_key: int = bittensor.__version_as_int__, wait_for_inclusion: bool = False, From 80ec4d6965fdd2f3a3a38eafd8bc9741d71ebc64 Mon Sep 17 00:00:00 2001 From: opendansor Date: Tue, 28 May 2024 14:48:39 -0700 Subject: [PATCH 11/12] Fix test. Normalize weights before passing as params. --- .../test_subtensor_integration.py | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/tests/integration_tests/test_subtensor_integration.py b/tests/integration_tests/test_subtensor_integration.py index 672f8437e6..8849aeb9db 100644 --- a/tests/integration_tests/test_subtensor_integration.py +++ b/tests/integration_tests/test_subtensor_integration.py @@ -21,12 +21,13 @@ from queue import Empty as QueueEmpty from unittest.mock import MagicMock, patch +import numpy as np import pytest from substrateinterface import Keypair -import numpy as np import bittensor from bittensor.mock import MockSubtensor +from bittensor.utils import weight_utils from bittensor.utils.balance import Balance from tests.helpers import ( _get_mock_coldkey, @@ -364,11 +365,14 @@ def test_commit_weights(self): weights = np.array([0.1, 0.2, 0.3, 0.4], dtype=np.float32) uids = np.array([1, 2, 3, 4], dtype=np.int64) salt = np.array([1, 2, 3, 4, 5, 6, 7, 8], dtype=np.int64) + weight_uids, weight_vals = weight_utils.convert_weights_and_uids_for_emit( + uids=uids, weights=weights + ) commit_hash = bittensor.utils.weight_utils.generate_weight_hash( address=self.wallet.hotkey.ss58_address, netuid=3, - uids=uids.tolist(), - values=weights.tolist(), + uids=weight_uids, + values=weight_vals, salt=salt.tolist(), version_key=0, ) @@ -388,11 +392,16 @@ def test_commit_weights_inclusion(self): weights = np.array([0.1, 0.2, 0.3, 0.4], dtype=np.float32) uids = np.array([1, 2, 3, 4], dtype=np.int64) salt = np.array([1, 2, 3, 4, 5, 6, 7, 8], dtype=np.int64) + + weight_uids, weight_vals = weight_utils.convert_weights_and_uids_for_emit( + uids=uids, weights=weights + ) + commit_hash = bittensor.utils.weight_utils.generate_weight_hash( address=self.wallet.hotkey.ss58_address, netuid=1, - uids=uids.tolist(), - values=weights.tolist(), + uids=weight_uids, + values=weight_vals, salt=salt.tolist(), version_key=0, ) @@ -417,11 +426,16 @@ def test_commit_weights_failed(self): weights = np.array([0.1, 0.2, 0.3, 0.4], dtype=np.float32) uids = np.array([1, 2, 3, 4], dtype=np.int64) salt = np.array([1, 2, 3, 4, 5, 6, 7, 8], dtype=np.int64) + + weight_uids, weight_vals = weight_utils.convert_weights_and_uids_for_emit( + uids=uids, weights=weights + ) + commit_hash = bittensor.utils.weight_utils.generate_weight_hash( address=self.wallet.hotkey.ss58_address, netuid=3, - uids=uids.tolist(), - values=weights.tolist(), + uids=weight_uids, + values=weight_vals, salt=salt.tolist(), version_key=0, ) From c0ca5da839dd12d3dc2d1141c18768354c7d896b Mon Sep 17 00:00:00 2001 From: opendansor Date: Tue, 28 May 2024 14:57:04 -0700 Subject: [PATCH 12/12] Fix test. Normalize weights before passing as params. Set byte length to 2. --- bittensor/utils/weight_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bittensor/utils/weight_utils.py b/bittensor/utils/weight_utils.py index c7e25cf6a5..42f0a88380 100644 --- a/bittensor/utils/weight_utils.py +++ b/bittensor/utils/weight_utils.py @@ -388,7 +388,7 @@ def generate_weight_hash( version_key = ScaleBytes(version_key.to_bytes(8, "little")) vec_salt = Vec(data=None, sub_type="U16") - vec_salt.value = [U16(ScaleBytes(salts.to_bytes(3, "little"))) for salts in salt] + vec_salt.value = [U16(ScaleBytes(salts.to_bytes(2, "little"))) for salts in salt] salt = ScaleBytes(vec_salt.encode().data) data = wallet_address + netuid + uids + values + salt + version_key