From 7a76e47601d56835650ca0e12eaf6338e3353a92 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 5 Oct 2022 09:20:29 -0400 Subject: [PATCH 01/27] adjust none end calculation --- bittensor/utils/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 1da54cab7e..ad796b37f6 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -285,8 +285,8 @@ def run(self): self.newBlockEvent.clear() # reset nonces to start from random point - nonce_start = self.update_interval * self.proc_num + random.randint( 0, nonce_limit ) - nonce_end = nonce_start + self.update_interval + nonce_start = self.TPB * self.update_interval * self.proc_num + random.randint( 0, nonce_limit ) + nonce_end = nonce_start + self.update_interval * self.TPB # Do a block of nonces solution, time = solve_for_nonce_block_cuda(self, nonce_start, self.update_interval, block_bytes, block_difficulty, self.limit, block_number, self.dev_id, self.TPB) From 049f05e4466902cc0d6dccc5c00051dae80a9235 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 5 Oct 2022 09:21:39 -0400 Subject: [PATCH 02/27] attempt to fix stop issue --- bittensor/utils/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index ad796b37f6..d4e30a40ab 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -518,6 +518,10 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = stopEvent.set() # stop all other processes status.stop() + # wait for rest to exit + for w in solvers: + w.join() + return solution def get_human_readable(num, suffix="H"): From 9bb48ff6f3aebd3ec9d32c26fd25a97227f9c296 Mon Sep 17 00:00:00 2001 From: camfairchild Date: Wed, 5 Oct 2022 17:12:20 -0400 Subject: [PATCH 03/27] modify stop --- bittensor/utils/__init__.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index d4e30a40ab..8482961849 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -518,8 +518,9 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = stopEvent.set() # stop all other processes status.stop() - # wait for rest to exit + # terminate and wait for sovlers to exit for w in solvers: + w.terminate() w.join() return solution @@ -701,15 +702,15 @@ def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: mu Block_hash: [bold white]{block_hash.encode('utf-8')}[/bold white]""" status.update(message.replace(" ", "")) - # exited while, found_solution contains the nonce or wallet is registered - if solution is not None: - stopEvent.set() # stop all other processes - status.stop() + # exited while, solution contains the nonce or wallet is registered + stopEvent.set() # stop all other processes + status.stop() - return solution + for w in solvers: # terminate and wait for all solvers + w.terminate() + w.join() - status.stop() - return None + return solution def create_pow( subtensor, wallet, cuda: bool = False, dev_id: Union[List[int], int] = 0, tpb: int = 256, num_processes: int = None, update_interval: int = None) -> Optional[Dict[str, Any]]: if cuda: From 3da133ec41e8da1ba53e76f7db903a58cf1ef65e Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 6 Oct 2022 22:43:09 -0400 Subject: [PATCH 04/27] update nonce_start by correct amount --- bittensor/utils/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 8482961849..86a62c73e7 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -275,7 +275,6 @@ def run(self): # Start at random nonce nonce_start = self.TPB * self.update_interval * self.proc_num + random.randint( 0, nonce_limit ) - nonce_end = nonce_start + self.update_interval * self.TPB while not self.stopEvent.is_set(): if self.newBlockEvent.is_set(): with self.check_block: @@ -286,7 +285,6 @@ def run(self): self.newBlockEvent.clear() # reset nonces to start from random point nonce_start = self.TPB * self.update_interval * self.proc_num + random.randint( 0, nonce_limit ) - nonce_end = nonce_start + self.update_interval * self.TPB # Do a block of nonces solution, time = solve_for_nonce_block_cuda(self, nonce_start, self.update_interval, block_bytes, block_difficulty, self.limit, block_number, self.dev_id, self.TPB) @@ -296,9 +294,8 @@ def run(self): # Send time self.time_queue.put_nowait(time) - nonce_start += self.update_interval * self.num_proc + nonce_start += self.update_interval * self.num_proc * self.TPB nonce_start = nonce_start % nonce_limit - nonce_end += self.update_interval * self.num_proc def solve_for_nonce_block_cuda(solver: CUDASolver, nonce_start: int, update_interval: int, block_bytes: bytes, difficulty: int, limit: int, block_number: int, dev_id: int, TPB: int) -> Tuple[Optional[POWSolution], int]: From 418b3a4fc0d47ccf2cdad2d50c7c01420554b431 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Fri, 7 Oct 2022 11:59:08 -0400 Subject: [PATCH 05/27] fix nonce init to only random and update --- bittensor/utils/__init__.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 86a62c73e7..78a90c00b2 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -241,11 +241,6 @@ def run(self): block_difficulty = registration_diff_unpack(self.curr_diff) self.newBlockEvent.clear() - # reset nonces to start from random point - # prevents the same nonces (for each block) from being tried by multiple processes - # also prevents the same nonces from being tried by multiple peers - nonce_start = random.randint( 0, nonce_limit ) - nonce_end = nonce_start + self.update_interval # Do a block of nonces solution, time = solve_for_nonce_block(self, nonce_start, nonce_end, block_bytes, block_difficulty, self.limit, block_number) @@ -255,8 +250,9 @@ def run(self): # Send time self.time_queue.put_nowait(time) - nonce_start += self.update_interval * self.num_proc - nonce_end += self.update_interval * self.num_proc + nonce_start = random.randint( 0, nonce_limit ) + nonce_end += self.update_interval + nonce_start = nonce_start % nonce_limit class CUDASolver(SolverBase): dev_id: int @@ -274,7 +270,7 @@ def run(self): nonce_limit = int(math.pow(2,64)) - 1 # Start at random nonce - nonce_start = self.TPB * self.update_interval * self.proc_num + random.randint( 0, nonce_limit ) + nonce_start = random.randint( 0, nonce_limit ) while not self.stopEvent.is_set(): if self.newBlockEvent.is_set(): with self.check_block: @@ -283,8 +279,6 @@ def run(self): block_difficulty = registration_diff_unpack(self.curr_diff) self.newBlockEvent.clear() - # reset nonces to start from random point - nonce_start = self.TPB * self.update_interval * self.proc_num + random.randint( 0, nonce_limit ) # Do a block of nonces solution, time = solve_for_nonce_block_cuda(self, nonce_start, self.update_interval, block_bytes, block_difficulty, self.limit, block_number, self.dev_id, self.TPB) @@ -293,8 +287,9 @@ def run(self): # Send time self.time_queue.put_nowait(time) - - nonce_start += self.update_interval * self.num_proc * self.TPB + + # increase nonce by number of nonces processed + nonce_start += self.update_interval * self.TPB nonce_start = nonce_start % nonce_limit From fed88a39bf363c8b5ccfca546d4106ad91cb0841 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Fri, 7 Oct 2022 12:06:54 -0400 Subject: [PATCH 06/27] fix update amount --- bittensor/utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 78a90c00b2..de5b256013 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -251,8 +251,8 @@ def run(self): self.time_queue.put_nowait(time) nonce_start = random.randint( 0, nonce_limit ) - nonce_end += self.update_interval nonce_start = nonce_start % nonce_limit + nonce_end = nonce_start + self.update_interval class CUDASolver(SolverBase): dev_id: int From 8b0883c0f6db65513d5f834fee5cf033bbeece08 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Fri, 7 Oct 2022 12:08:08 -0400 Subject: [PATCH 07/27] add start values --- bittensor/utils/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index de5b256013..42a8cc27df 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -264,10 +264,10 @@ def __init__(self, proc_num, num_proc, update_interval, time_queue, solution_que self.TPB = TPB def run(self): - block_number: int - block_bytes: bytes - block_difficulty: int - nonce_limit = int(math.pow(2,64)) - 1 + block_number: int = 0 # dummy value + block_bytes: bytes = b'0' * 32 # dummy value + block_difficulty: int = int(math.pow(2,64)) - 1 # dummy value + nonce_limit = int(math.pow(2,64)) - 1 # U64MAX # Start at random nonce nonce_start = random.randint( 0, nonce_limit ) From d6510d5a3bec04fd54268b327561ff75184ef4cf Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Fri, 7 Oct 2022 12:12:51 -0400 Subject: [PATCH 08/27] add test --- .../bittensor_tests/utils/test_utils.py | 126 ++++++++---------- 1 file changed, 59 insertions(+), 67 deletions(-) diff --git a/tests/unit_tests/bittensor_tests/utils/test_utils.py b/tests/unit_tests/bittensor_tests/utils/test_utils.py index feb1807250..669877d62e 100644 --- a/tests/unit_tests/bittensor_tests/utils/test_utils.py +++ b/tests/unit_tests/bittensor_tests/utils/test_utils.py @@ -1,26 +1,24 @@ import binascii import hashlib -import unittest -import bittensor -import sys +import math +import multiprocessing +import os +import random import subprocess +import sys import time -import pytest -import os -import random -import torch -import multiprocessing +import unittest +from sys import platform from types import SimpleNamespace +from unittest.mock import MagicMock, patch -from sys import platform -from substrateinterface.base import Keypair +import bittensor +import pytest +import torch from _pytest.fixtures import fixture +from bittensor.utils import CUDASolver from loguru import logger - -from types import SimpleNamespace - -from unittest.mock import MagicMock, patch - +from substrateinterface.base import Keypair @fixture(scope="function") @@ -400,60 +398,54 @@ class MockException(Exception): assert call1[1]['call_function'] == 'register' call_params = call1[1]['call_params'] assert call_params['nonce'] == mock_result['nonce'] - - -def test_pow_called_for_cuda(): - class MockException(Exception): - pass - mock_compose_call = MagicMock(side_effect=MockException) - - mock_subtensor = bittensor.subtensor(_mock=True) - mock_subtensor.neuron_for_pubkey=MagicMock(is_null=True) - mock_subtensor.substrate = MagicMock( - __enter__= MagicMock(return_value=MagicMock( - compose_call=mock_compose_call - )), - __exit__ = MagicMock(return_value=None), - ) - - mock_wallet = SimpleNamespace( - hotkey=SimpleNamespace( - ss58_address='' - ), - coldkeypub=SimpleNamespace( - ss58_address='' - ) - ) - mock_result = { - "block_number": 1, - 'nonce': random.randint(0, pow(2, 32)), - 'work': b'\x00' * 64, - } +class TestCUDASolverRun(unittest.TestCase): + def test_multi_cuda_run_updates_nonce_start(self): + class MockException(Exception): + pass + + TPB: int = 512 + update_interval: int = 70_000 + nonce_limit: int = int(math.pow(2, 64)) - 1 + + mock_solver_self = MagicMock( + spec=CUDASolver, + TPB=TPB, + dev_id=0, + update_interval=update_interval, + stopEvent=MagicMock(is_set=MagicMock(return_value=False)), + newBlockEvent=MagicMock(is_set=MagicMock(return_value=False)), + time_queue=MagicMock(put_nowait=MagicMock()), + limit=10000 + ) - with patch('bittensor.utils.POWNotStale', return_value=True) as mock_pow_not_stale: - with patch('torch.cuda.is_available', return_value=True) as mock_cuda_available: - with patch('bittensor.utils.create_pow', return_value=mock_result) as mock_create_pow: - with patch('bittensor.utils.hex_bytes_to_u8_list', return_value=b''): - - # Should exit early - with pytest.raises(MockException): - mock_subtensor.register(mock_wallet, cuda=True, prompt=False) - - mock_pow_not_stale.assert_called_once() - mock_create_pow.assert_called_once() - mock_cuda_available.assert_called_once() - - call0 = mock_pow_not_stale.call_args - assert call0[0][0] == mock_subtensor - assert call0[0][1] == mock_result - - mock_compose_call.assert_called_once() - call1 = mock_compose_call.call_args - assert call1[1]['call_function'] == 'register' - call_params = call1[1]['call_params'] - assert call_params['nonce'] == mock_result['nonce'] + + with patch('bittensor.utils.solve_for_nonce_block_cuda', + side_effect=[(None, 10.0), MockException] # first call returns mocked no solution, second call raises exception + ) as mock_solve_for_nonce_block_cuda: + + # Should exit early + with pytest.raises(MockException): + CUDASolver.run(mock_solver_self) + + mock_solve_for_nonce_block_cuda.assert_called() + calls = mock_solve_for_nonce_block_cuda.call_args_list + self.assertEqual(len(calls), 2, f"solve_for_nonce_block_cuda was called {len(calls)}. Expected 2") # called only twice + + # args, kwargs + args_call_0, _ = calls[0] + initial_nonce_start: int = args_call_0[1] # second arg should be nonce_start + self.assertIsInstance(initial_nonce_start, int) + + args_call_1, _ = calls[1] + nonce_start_after_iteration: int = args_call_1[1] # second arg should be nonce_start + self.assertIsInstance(nonce_start_after_iteration, int) + + # verify nonce_start is updated after each iteration + self.assertNotEqual(nonce_start_after_iteration, initial_nonce_start, "nonce_start was not updated after iteration") + ## Should incerase by the number of nonces tried == TPB * update_interval + self.assertEqual(nonce_start_after_iteration, (initial_nonce_start + update_interval * TPB) % nonce_limit, "nonce_start was not updated by the correct amount") if __name__ == "__main__": - test_solve_for_difficulty_fast_registered_already() + unittest.main() From bf005159868aa4812347ba4d6984aef81b4a30ac Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Sun, 9 Oct 2022 17:29:53 -0400 Subject: [PATCH 09/27] try different hashrate calc --- bittensor/utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 42a8cc27df..85e0232ca0 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -683,7 +683,7 @@ def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: mu if num_time > 0: time_avg = time_total / num_time - itrs_per_sec = TPB*update_interval*num_processes / time_avg + itrs_per_sec = TPB*update_interval / time_avg time_since = time.time() - start_time message = f"""Solving From 1d0ecd2819090a440684fb689aa70372f463bec4 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Tue, 11 Oct 2022 10:16:28 -0400 Subject: [PATCH 10/27] try EWMA for hash_rate --- bittensor/utils/__init__.py | 39 +++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 85e0232ca0..ee707741a8 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -8,6 +8,7 @@ import time from dataclasses import dataclass from queue import Empty +from tracemalloc import start from typing import Any, Dict, List, Optional, Tuple, Union import backoff @@ -639,10 +640,18 @@ def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: mu for w in solvers: w.start() # start the solver processes - start_time = time.time() - time_since = 0.0 + start_time = time.time() # time that the registration started + time_last = start_time # time that the last work blocks completed + solution = None - itrs_per_sec = 0 + hash_rate = 0 # EWMA hash_rate (H/s) + + n = 5 # number of samples to keep + alpha_ = 0.70 # EWMA alpha for hash_rate + + hash_rates = [0] * n # The last n true hash_rates + weights = [alpha_ ** i for i in range(n)] # weights decay by alpha + while not wallet.is_registered(subtensor): # Wait until a solver finds a solution try: @@ -668,28 +677,34 @@ def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: mu # Set new block events for each solver for w in solvers: w.newBlockEvent.set() - - # Get times for each solver - time_total = 0 + num_time = 0 + # Get times for each solver for _ in solvers: try: - time_ = time_queue.get_nowait() - time_total += time_ + _ = time_queue.get_nowait() num_time += 1 except Empty: break + time_since = time.time() - start_time if num_time > 0: - time_avg = time_total / num_time - itrs_per_sec = TPB*update_interval / time_avg - time_since = time.time() - start_time + time_since_last = time.time() - time_last + # create EWMA of the hash_rate to make measure more robust + hash_rate_ = ((num_time * TPB * update_interval) / time_since_last) + hash_rates.append(hash_rate_) + hash_rates.pop(0) # remove the 0th data point + hash_rate = sum([hash_rates[i]*weights[i] for i in range(n)])/(sum(weights)) + + # update time last to now + time_last = time.time() + message = f"""Solving time spent: {time_since} Difficulty: [bold white]{millify(difficulty)}[/bold white] - Iters: [bold white]{get_human_readable(int(itrs_per_sec), 'H')}/s[/bold white] + Iters: [bold white]{get_human_readable(int(hash_rate), 'H')}/s[/bold white] Block: [bold white]{block_number}[/bold white] Block_hash: [bold white]{block_hash.encode('utf-8')}[/bold white]""" status.update(message.replace(" ", "")) From a20e992408065a21e04cfceb1edfb59f9c43bad5 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Tue, 11 Oct 2022 10:17:01 -0400 Subject: [PATCH 11/27] oops bad import --- bittensor/utils/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index ee707741a8..583e7c66ec 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -8,7 +8,6 @@ import time from dataclasses import dataclass from queue import Empty -from tracemalloc import start from typing import Any, Dict, List, Optional, Tuple, Union import backoff From 241f6218ab1a6159a1babfd46990b52c3aeb6c70 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Tue, 11 Oct 2022 11:44:45 -0400 Subject: [PATCH 12/27] change name to worker --- bittensor/utils/__init__.py | 41 +++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 583e7c66ec..253c07cd28 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -434,11 +434,11 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = update_curr_block(curr_diff, curr_block, curr_block_num, block_number, block_bytes, difficulty, check_block) # Set new block events for each solver to start - for w in solvers: - w.newBlockEvent.set() + for worker in solvers: + worker.newBlockEvent.set() - for w in solvers: - w.start() # start the solver processes + for worker in solvers: + worker.start() # start the solver processes start_time = time.time() solution = None @@ -467,8 +467,8 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = update_curr_block(curr_diff, curr_block, curr_block_num, block_number, block_bytes, difficulty, check_block) # Set new block events for each solver - for w in solvers: - w.newBlockEvent.set() + for worker in solvers: + worker.newBlockEvent.set() # Get times for each solver time_total = 0 @@ -510,10 +510,10 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = stopEvent.set() # stop all other processes status.stop() - # terminate and wait for sovlers to exit - for w in solvers: - w.terminate() - w.join() + # terminate and wait for solvers/workers to exit + for worker in solvers: + worker.terminate() + worker.join() return solution @@ -633,11 +633,11 @@ def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: mu update_curr_block(block_number, block_bytes, difficulty, check_block) # Set new block events for each solver to start - for w in solvers: - w.newBlockEvent.set() + for worker in solvers: + worker.newBlockEvent.set() - for w in solvers: - w.start() # start the solver processes + for worker in solvers: + worker.start() # start the solver processes start_time = time.time() # time that the registration started time_last = start_time # time that the last work blocks completed @@ -674,9 +674,10 @@ def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: mu update_curr_block(block_number, block_bytes, difficulty, check_block) # Set new block events for each solver - for w in solvers: - w.newBlockEvent.set() - + + for worker in solvers: + worker.newBlockEvent.set() + num_time = 0 # Get times for each solver for _ in solvers: @@ -712,9 +713,9 @@ def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: mu stopEvent.set() # stop all other processes status.stop() - for w in solvers: # terminate and wait for all solvers - w.terminate() - w.join() + for worker in solvers: # terminate and wait for all solvers + worker.terminate() + worker.join() return solution From cbd632c2b023b8e88470348da17386075abfcf30 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Tue, 11 Oct 2022 11:49:04 -0400 Subject: [PATCH 13/27] extract helper and modify comment --- bittensor/utils/__init__.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 253c07cd28..143087d16f 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -433,7 +433,7 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = # Set to current block update_curr_block(curr_diff, curr_block, curr_block_num, block_number, block_bytes, difficulty, check_block) - # Set new block events for each solver to start + # Set new block events for each solver to start at the initial block for worker in solvers: worker.newBlockEvent.set() @@ -510,10 +510,8 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = stopEvent.set() # stop all other processes status.stop() - # terminate and wait for solvers/workers to exit - for worker in solvers: - worker.terminate() - worker.join() + # terminate and wait for all solvers to exit + terminate_workers_and_wait_for_exit(solvers) return solution @@ -632,7 +630,7 @@ def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: mu # Set to current block update_curr_block(block_number, block_bytes, difficulty, check_block) - # Set new block events for each solver to start + # Set new block events for each solver to start at the initial block for worker in solvers: worker.newBlockEvent.set() @@ -713,12 +711,17 @@ def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: mu stopEvent.set() # stop all other processes status.stop() - for worker in solvers: # terminate and wait for all solvers - worker.terminate() - worker.join() + # terminate and wait for all solvers to exit + terminate_workers_and_wait_for_exit(solvers) return solution +def terminate_workers_and_wait_for_exit(workers: List[multiprocessing.Process]) -> None: + for worker in workers: + worker.terminate() + worker.join() + + def create_pow( subtensor, wallet, cuda: bool = False, dev_id: Union[List[int], int] = 0, tpb: int = 256, num_processes: int = None, update_interval: int = None) -> Optional[Dict[str, Any]]: if cuda: solution: POWSolution = solve_for_difficulty_fast_cuda( subtensor, wallet, dev_id=dev_id, TPB=tpb, update_interval=update_interval ) From 4f46cd95512a97480255efd02d4062bcd596a9f8 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Tue, 11 Oct 2022 16:08:15 -0400 Subject: [PATCH 14/27] fix time now --- bittensor/utils/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 143087d16f..a9d8482281 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -686,17 +686,20 @@ def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: mu except Empty: break - time_since = time.time() - start_time - if num_time > 0: - time_since_last = time.time() - time_last + time_now = time.time() # get current time + time_since_last = time_now - time_last # get time since last work block(s) + if num_time > 0 and time_since_last > 0.0: # create EWMA of the hash_rate to make measure more robust - hash_rate_ = ((num_time * TPB * update_interval) / time_since_last) + + hash_rate_ = (num_time * TPB * update_interval) / time_since_last hash_rates.append(hash_rate_) hash_rates.pop(0) # remove the 0th data point hash_rate = sum([hash_rates[i]*weights[i] for i in range(n)])/(sum(weights)) # update time last to now - time_last = time.time() + time_last = time_now + + time_since = time_now - start_time message = f"""Solving From 4e86b5bd605cb868435bdaf274c912e195862923 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Tue, 11 Oct 2022 17:24:59 -0400 Subject: [PATCH 15/27] catch Full --- bittensor/utils/__init__.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index a9d8482281..09929ac689 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -7,7 +7,7 @@ import random import time from dataclasses import dataclass -from queue import Empty +from queue import Empty, Full from typing import Any, Dict, List, Optional, Tuple, Union import backoff @@ -247,8 +247,11 @@ def run(self): if solution is not None: self.solution_queue.put(solution) - # Send time - self.time_queue.put_nowait(time) + try: + # Send time + self.time_queue.put_nowait(time) + except Full: + pass nonce_start = random.randint( 0, nonce_limit ) nonce_start = nonce_start % nonce_limit @@ -284,9 +287,12 @@ def run(self): solution, time = solve_for_nonce_block_cuda(self, nonce_start, self.update_interval, block_bytes, block_difficulty, self.limit, block_number, self.dev_id, self.TPB) if solution is not None: self.solution_queue.put(solution) - - # Send time - self.time_queue.put_nowait(time) + + try: + # Send time + self.time_queue.put_nowait(time) + except Full: + pass # increase nonce by number of nonces processed nonce_start += self.update_interval * self.TPB From bd04152e8e86215e20f3499beed3873c6e8fd232 Mon Sep 17 00:00:00 2001 From: Eugene-hu <85906264+Eugene-hu@users.noreply.github.com> Date: Tue, 11 Oct 2022 14:44:39 -0700 Subject: [PATCH 16/27] Prometheus bug fix (#942) * local train bug fix * normalization update * fix tests * remove test * updated normalization * Naming changes, bug fixes * subtensor update for max clip * max weight to a million * Fixes for ordering and comments * additional tests * string fix * numerical stability and testing updates * minor update for division by zero * Naming and spacing fixes * epsilon update * small fix * additional subtensor parameters * remove print * help string fixes * small bug fix --- bittensor/_neuron/text/core_validator/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index 8e3002d3da..9916757b20 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -398,7 +398,6 @@ def run_epoch( self ): self.prometheus_gauges.labels("sequence_length").set( sequence_length ) self.prometheus_gauges.labels("validation_len").set( validation_len ) self.prometheus_gauges.labels("min_allowed_weights").set( min_allowed_weights ) - self.prometheus_gauges.labels("max_allowed_ratio").set( max_allowed_ratio ) self.prometheus_gauges.labels("blocks_per_epoch").set( blocks_per_epoch ) self.prometheus_gauges.labels("epochs_until_reset").set( epochs_until_reset ) From 3a76a790cfc34cfcdb7020dd2055320b0ef63d41 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Tue, 11 Oct 2022 18:44:52 -0400 Subject: [PATCH 17/27] use a finished queue instead of times --- bittensor/utils/__init__.py | 125 +++++++++++++++++++++--------------- 1 file changed, 72 insertions(+), 53 deletions(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 09929ac689..6a5b825cf3 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -158,8 +158,8 @@ class SolverBase(multiprocessing.Process): best_queue: multiprocessing.Queue The queue to put the best nonce the process has found during the pow solve. New nonces are added each update_interval. - time_queue: multiprocessing.Queue - The queue to put the time the process took to finish each update_interval. + finished_queue: multiprocessing.Queue + The queue to put the process number when a process finishes each update_interval. Used for calculating the average time per update_interval across all processes. solution_queue: multiprocessing.Queue The queue to put the solution the process has found during the pow solve. @@ -193,7 +193,7 @@ class SolverBase(multiprocessing.Process): num_proc: int update_interval: int best_queue: Optional[multiprocessing.Queue] - time_queue: multiprocessing.Queue + finished_queue: multiprocessing.Queue solution_queue: multiprocessing.Queue newBlockEvent: multiprocessing.Event stopEvent: multiprocessing.Event @@ -203,13 +203,13 @@ class SolverBase(multiprocessing.Process): check_block: multiprocessing.Lock limit: int - def __init__(self, proc_num, num_proc, update_interval, best_queue, time_queue, solution_queue, stopEvent, curr_block, curr_block_num, curr_diff, check_block, limit): + def __init__(self, proc_num, num_proc, update_interval, best_queue, finished_queue, solution_queue, stopEvent, curr_block, curr_block_num, curr_diff, check_block, limit): multiprocessing.Process.__init__(self) self.proc_num = proc_num self.num_proc = num_proc self.update_interval = update_interval self.best_queue = best_queue - self.time_queue = time_queue + self.finished_queue = finished_queue self.solution_queue = solution_queue self.newBlockEvent = multiprocessing.Event() self.newBlockEvent.clear() @@ -243,13 +243,13 @@ def run(self): self.newBlockEvent.clear() # Do a block of nonces - solution, time = solve_for_nonce_block(self, nonce_start, nonce_end, block_bytes, block_difficulty, self.limit, block_number) + solution = solve_for_nonce_block(self, nonce_start, nonce_end, block_bytes, block_difficulty, self.limit, block_number) if solution is not None: self.solution_queue.put(solution) try: # Send time - self.time_queue.put_nowait(time) + self.finished_queue.put_nowait(self.proc_num) except Full: pass @@ -261,8 +261,8 @@ class CUDASolver(SolverBase): dev_id: int TPB: int - def __init__(self, proc_num, num_proc, update_interval, time_queue, solution_queue, stopEvent, curr_block, curr_block_num, curr_diff, check_block, limit, dev_id: int, TPB: int): - super().__init__(proc_num, num_proc, update_interval, None, time_queue, solution_queue, stopEvent, curr_block, curr_block_num, curr_diff, check_block, limit) + def __init__(self, proc_num, num_proc, update_interval, finished_queue, solution_queue, stopEvent, curr_block, curr_block_num, curr_diff, check_block, limit, dev_id: int, TPB: int): + super().__init__(proc_num, num_proc, update_interval, None, finished_queue, solution_queue, stopEvent, curr_block, curr_block_num, curr_diff, check_block, limit) self.dev_id = dev_id self.TPB = TPB @@ -284,13 +284,14 @@ def run(self): self.newBlockEvent.clear() # Do a block of nonces - solution, time = solve_for_nonce_block_cuda(self, nonce_start, self.update_interval, block_bytes, block_difficulty, self.limit, block_number, self.dev_id, self.TPB) + solution = solve_for_nonce_block_cuda(self, nonce_start, self.update_interval, block_bytes, block_difficulty, self.limit, block_number, self.dev_id, self.TPB) if solution is not None: self.solution_queue.put(solution) - + try: - # Send time - self.time_queue.put_nowait(time) + # Signal that a nonce_block was finished using queue + # send our proc_num + self.finished_queue.put(self.proc_num) except Full: pass @@ -299,9 +300,8 @@ def run(self): nonce_start = nonce_start % nonce_limit -def solve_for_nonce_block_cuda(solver: CUDASolver, nonce_start: int, update_interval: int, block_bytes: bytes, difficulty: int, limit: int, block_number: int, dev_id: int, TPB: int) -> Tuple[Optional[POWSolution], int]: - start = time.time() - +def solve_for_nonce_block_cuda(solver: CUDASolver, nonce_start: int, update_interval: int, block_bytes: bytes, difficulty: int, limit: int, block_number: int, dev_id: int, TPB: int) -> Optional[POWSolution]: + """Tries to solve the POW on a CUDA device for a block of nonces (nonce_start, nonce_start + update_interval * TPB""" solution, seal = solve_cuda(nonce_start, update_interval, TPB, @@ -312,21 +312,16 @@ def solve_for_nonce_block_cuda(solver: CUDASolver, nonce_start: int, update_inte dev_id) if (solution != -1): - # Check if solution is valid - # Attempt to reset CUDA device - #reset_cuda() - - #print(f"{solver.proc_num} on cuda:{solver.dev_id} found a solution: {solution}, {block_number}, {str(block_bytes)}, {str(seal)}, {difficulty}") - # Found a solution, save it. - return POWSolution(solution, block_number, difficulty, seal), time.time() - start + # Check if solution is valid (i.e. not -1) + return POWSolution(solution, block_number, difficulty, seal) - return None, time.time() - start + return None -def solve_for_nonce_block(solver: Solver, nonce_start: int, nonce_end: int, block_bytes: bytes, difficulty: int, limit: int, block_number: int) -> Tuple[Optional[POWSolution], int]: +def solve_for_nonce_block(solver: Solver, nonce_start: int, nonce_end: int, block_bytes: bytes, difficulty: int, limit: int, block_number: int) -> Optional[POWSolution]: + """Tries to solve the POW for a block of nonces (nonce_start, nonce_end)""" best_local = float('inf') best_seal_local = [0]*32 - start = time.time() for nonce in range(nonce_start, nonce_end): # Create seal. nonce_bytes = binascii.hexlify(nonce.to_bytes(8, 'little')) @@ -340,7 +335,7 @@ def solve_for_nonce_block(solver: Solver, nonce_start: int, nonce_end: int, bloc product = seal_number * difficulty if product < limit: # Found a solution, save it. - return POWSolution(nonce, block_number, difficulty, seal), time.time() - start + return POWSolution(nonce, block_number, difficulty, seal) if (product - limit) < best_local: best_local = product - limit @@ -348,7 +343,7 @@ def solve_for_nonce_block(solver: Solver, nonce_start: int, nonce_end: int, bloc # Send best solution to best queue. solver.best_queue.put((best_local, best_seal_local)) - return None, time.time() - start + return None def registration_diff_unpack(packed_diff: multiprocessing.Array) -> int: @@ -361,6 +356,9 @@ def registration_diff_pack(diff: int, packed_diff: multiprocessing.Array): packed_diff[0] = diff >> 32 packed_diff[1] = diff & 0xFFFFFFFF # low 32 bits +def calculate_hash_rate() -> int: + pass + def update_curr_block(curr_diff: multiprocessing.Array, curr_block: multiprocessing.Array, curr_block_num: multiprocessing.Value, block_number: int, block_bytes: bytes, diff: int, lock: multiprocessing.Lock): with lock: @@ -421,11 +419,11 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = stopEvent.clear() best_queue = multiprocessing.Queue() solution_queue = multiprocessing.Queue() - time_queue = multiprocessing.Queue() + finished_queue = multiprocessing.Queue() check_block = multiprocessing.Lock() # Start consumers - solvers = [ Solver(i, num_processes, update_interval, best_queue, time_queue, solution_queue, stopEvent, curr_block, curr_block_num, curr_diff, check_block, limit) + solvers = [ Solver(i, num_processes, update_interval, best_queue, finished_queue, solution_queue, stopEvent, curr_block, curr_block_num, curr_diff, check_block, limit) for i in range(num_processes) ] # Get first block @@ -449,7 +447,15 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = start_time = time.time() solution = None best_seal = None - itrs_per_sec = 0 + + hash_rate = 0 # EWMA hash_rate (H/s) + + n = 5 # number of samples to keep + alpha_ = 0.70 # EWMA alpha for hash_rate + + hash_rates = [0] * n # The last n true hash_rates + weights = [alpha_ ** i for i in range(n)] # weights decay by alpha + while not wallet.is_registered(subtensor): # Wait until a solver finds a solution try: @@ -475,22 +481,32 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = # Set new block events for each solver for worker in solvers: worker.newBlockEvent.set() - - # Get times for each solver - time_total = 0 + + # find hash_rate num_time = 0 - - for _ in solvers: + for _ in range(len(solvers)*2): try: - time_total += time_queue.get_nowait() + proc_num = finished_queue.get(timeout=0.1) num_time += 1 + except Empty: - break + # no more times + continue + + time_now = time.time() # get current time + time_since_last = time_now - time_last # get time since last work block(s) + if num_time > 0 and time_since_last > 0.0: + # create EWMA of the hash_rate to make measure more robust + + hash_rate_ = (num_time * update_interval) / time_since_last + hash_rates.append(hash_rate_) + hash_rates.pop(0) # remove the 0th data point + hash_rate = sum([hash_rates[i]*weights[i] for i in range(n)])/(sum(weights)) + + # update time last to now + time_last = time_now - # Calculate average time per solver for the update_interval - if num_time > 0: - time_avg = time_total / num_time - itrs_per_sec = update_interval*num_processes / time_avg + time_since = time_now - start_time # get best solution from each solver using the best_queue for _ in solvers: @@ -504,9 +520,9 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = break message = f"""Solving - time spent: {time.time() - start_time} + time spent: {time_since} Difficulty: [bold white]{millify(difficulty)}[/bold white] - Iters: [bold white]{get_human_readable(int(itrs_per_sec), 'H')}/s[/bold white] + Iters: [bold white]{get_human_readable(int(hash_rate), 'H')}/s[/bold white] Block: [bold white]{block_number}[/bold white] Block_hash: [bold white]{block_hash.encode('utf-8')}[/bold white] Best: [bold white]{binascii.hexlify(bytes(best_seal) if best_seal else bytes(0))}[/bold white]""" @@ -565,7 +581,7 @@ def __exit__(self, *args): multiprocessing.set_start_method(self._old_start_method, force=True) -def solve_for_difficulty_fast_cuda( subtensor: 'bittensor.Subtensor', wallet: 'bittensor.Wallet', update_interval: int = 50_000, TPB: int = 512, dev_id: Union[List[int], int] = 0, use_kernel_launch_optimization: bool = False ) -> Optional[POWSolution]: +def solve_for_difficulty_fast_cuda( subtensor: 'bittensor.Subtensor', wallet: 'bittensor.Wallet', update_interval: int = 50_000, TPB: int = 512, dev_id: Union[List[int], int] = 0 ) -> Optional[POWSolution]: """ Solves the registration fast using CUDA Args: @@ -616,15 +632,17 @@ def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: mu stopEvent = multiprocessing.Event() stopEvent.clear() solution_queue = multiprocessing.Queue() - time_queue = multiprocessing.Queue() + finished_queue = multiprocessing.Queue() check_block = multiprocessing.Lock() - # Start consumers + # Start workers + ## Create a worker per CUDA device num_processes = len(dev_id) - ## Create one consumer per GPU - solvers = [ CUDASolver(i, num_processes, update_interval, time_queue, solution_queue, stopEvent, curr_block, curr_block_num, curr_diff, check_block, limit, dev_id[i], TPB) + + solvers = [ CUDASolver(i, num_processes, update_interval, finished_queue, solution_queue, stopEvent, curr_block, curr_block_num, curr_diff, check_block, limit, dev_id[i], TPB) for i in range(num_processes) ] + # Get first block block_number = subtensor.get_current_block() difficulty = subtensor.difficulty @@ -684,13 +702,14 @@ def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: mu num_time = 0 # Get times for each solver - for _ in solvers: + for _ in range(len(solvers)*2): try: - _ = time_queue.get_nowait() + proc_num = finished_queue.get(timeout=0.1) num_time += 1 - + except Empty: - break + # no more times + continue time_now = time.time() # get current time time_since_last = time_now - time_last # get time since last work block(s) From 9ba253fafa8f2f76ae0b71e38c95624e8d646da0 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Tue, 11 Oct 2022 18:48:16 -0400 Subject: [PATCH 18/27] move constants to function params --- bittensor/utils/__init__.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 6a5b825cf3..4fde6c6e88 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -374,7 +374,7 @@ def get_cpu_count(): # OSX does not have sched_getaffinity return os.cpu_count() -def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = None, update_interval: Optional[int] = None ) -> Optional[POWSolution]: +def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = None, update_interval: Optional[int] = None, n_samples: int = 5, alpha_: float = 0.70 ) -> Optional[POWSolution]: """ Solves the POW for registration using multiprocessing. Args: @@ -386,6 +386,12 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = Number of processes to use. update_interval: int Number of nonces to solve before updating block information. + n_samples: int + The number of samples of the hash_rate to keep for the EWMA + alpha_: float + The alpha for the EWMA for the hash_rate calculation + + Note: The hash rate is calculated as an exponentially weighted moving average in order to make the measure more robust. Note: - We can also modify the update interval to do smaller blocks of work, while still updating the block information after a different number of nonces, @@ -450,9 +456,6 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = hash_rate = 0 # EWMA hash_rate (H/s) - n = 5 # number of samples to keep - alpha_ = 0.70 # EWMA alpha for hash_rate - hash_rates = [0] * n # The last n true hash_rates weights = [alpha_ ** i for i in range(n)] # weights decay by alpha @@ -581,7 +584,7 @@ def __exit__(self, *args): multiprocessing.set_start_method(self._old_start_method, force=True) -def solve_for_difficulty_fast_cuda( subtensor: 'bittensor.Subtensor', wallet: 'bittensor.Wallet', update_interval: int = 50_000, TPB: int = 512, dev_id: Union[List[int], int] = 0 ) -> Optional[POWSolution]: +def solve_for_difficulty_fast_cuda( subtensor: 'bittensor.Subtensor', wallet: 'bittensor.Wallet', update_interval: int = 50_000, TPB: int = 512, dev_id: Union[List[int], int] = 0, n: int = 5, alpha_: float = 0.70 ) -> Optional[POWSolution]: """ Solves the registration fast using CUDA Args: @@ -595,6 +598,12 @@ def solve_for_difficulty_fast_cuda( subtensor: 'bittensor.Subtensor', wallet: 'b The number of threads per block. CUDA param that should match the GPU capability dev_id: Union[List[int], int] The CUDA device IDs to execute the registration on, either a single device or a list of devices + n_samples: int + The number of samples of the hash_rate to keep for the EWMA + alpha_: float + The alpha for the EWMA for the hash_rate calculation + + Note: The hash rate is calculated as an exponentially weighted moving average in order to make the measure more robust. """ if isinstance(dev_id, int): dev_id = [dev_id] @@ -667,9 +676,6 @@ def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: mu solution = None hash_rate = 0 # EWMA hash_rate (H/s) - n = 5 # number of samples to keep - alpha_ = 0.70 # EWMA alpha for hash_rate - hash_rates = [0] * n # The last n true hash_rates weights = [alpha_ ** i for i in range(n)] # weights decay by alpha From f09a2f0051a7c830277fa60f7a142ebfccb0c205 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Tue, 11 Oct 2022 22:20:56 -0400 Subject: [PATCH 19/27] [Fix] only reregister if flag is set (#937) * add test for expected reregister behaviour * add fix * pass passed args into earlier parse * fix test by using args * exit before actual register * use strtobool Co-authored-by: Unconst <32490803+unconst@users.noreply.github.com> --- bittensor/_cli/cli_impl.py | 3 ++- bittensor/_config/__init__.py | 2 +- bittensor/_wallet/__init__.py | 3 ++- tests/integration_tests/test_cli.py | 32 +++++++++++++++++++++++++---- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/bittensor/_cli/cli_impl.py b/bittensor/_cli/cli_impl.py index 0425504486..369da862be 100644 --- a/bittensor/_cli/cli_impl.py +++ b/bittensor/_cli/cli_impl.py @@ -196,7 +196,8 @@ def run_miner ( self ): wallet.coldkeypub # Check registration - self.register() + ## Will exit if --wallet.reregister is False + wallet.reregister() # Run miner. if self.config.model == 'core_server': diff --git a/bittensor/_config/__init__.py b/bittensor/_config/__init__.py index b94e357544..76b4eff293 100644 --- a/bittensor/_config/__init__.py +++ b/bittensor/_config/__init__.py @@ -70,7 +70,7 @@ def __new__( cls, parser: ArgumentParser = None, strict: bool = False, args: Opt # 1.1 Optionally load defaults if the --config is set. try: - config_file_path = str(os.getcwd()) + '/' + vars(parser.parse_known_args()[0])['config'] + config_file_path = str(os.getcwd()) + '/' + vars(parser.parse_known_args(args)[0])['config'] except Exception as e: config_file_path = None diff --git a/bittensor/_wallet/__init__.py b/bittensor/_wallet/__init__.py index 4080ad8cf2..3f83f6b40d 100644 --- a/bittensor/_wallet/__init__.py +++ b/bittensor/_wallet/__init__.py @@ -19,6 +19,7 @@ import argparse import copy +from distutils.util import strtobool import os import bittensor @@ -114,7 +115,7 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ): parser.add_argument('--' + prefix_str + 'wallet.hotkeys', '--' + prefix_str + 'wallet.exclude_hotkeys', required=False, action='store', default=bittensor.defaults.wallet.hotkeys, type=str, nargs='*', help='''Specify the hotkeys by name. (e.g. hk1 hk2 hk3)''') parser.add_argument('--' + prefix_str + 'wallet.all_hotkeys', required=False, action='store_true', default=bittensor.defaults.wallet.all_hotkeys, help='''To specify all hotkeys. Specifying hotkeys will exclude them from this all.''') - parser.add_argument('--' + prefix_str + 'wallet.reregister', required=False, action='store', default=bittensor.defaults.wallet.reregister, type=bool, help='''Whether to reregister the wallet if it is not already registered.''') + parser.add_argument('--' + prefix_str + 'wallet.reregister', required=False, default=bittensor.defaults.wallet.reregister, type=lambda x: bool(strtobool(x)), help='''Whether to reregister the wallet if it is not already registered.''') except argparse.ArgumentError as e: pass diff --git a/tests/integration_tests/test_cli.py b/tests/integration_tests/test_cli.py index febbcbf5e7..4b8a7985d4 100644 --- a/tests/integration_tests/test_cli.py +++ b/tests/integration_tests/test_cli.py @@ -1354,8 +1354,32 @@ def test_btcli_help(): assert 'overview' in help_out assert 'run' in help_out +class TestCLIUsingArgs(unittest.TestCase): + """ + Test the CLI by passing args directly to the bittensor.cli factory + """ + def test_run_reregister_false(self): + """ + Verify that the btcli run command does not reregister a not registered wallet + if --wallet.reregister is False + """ + + with patch('bittensor.Wallet.is_registered', MagicMock(return_value=False)) as mock_wallet_is_reg: # Wallet is not registered + with patch('bittensor.Subtensor.register', MagicMock(side_effect=Exception("shouldn't register during test"))): + with pytest.raises(SystemExit): + cli = bittensor.cli(args=[ + 'run', + '--wallet.name', 'mock', + '--wallet.hotkey', 'mock_hotkey', + '--wallet._mock', 'True', + '--subtensor.network', 'mock', + '--subtensor._mock', 'True', + '--no_prompt', + '--wallet.reregister', 'False' # Don't reregister + ]) + cli.run() + + args, kwargs = mock_wallet_is_reg.call_args + # args[0] should be self => the wallet + assert args[0].config.wallet.reregister == False -if __name__ == "__main__": - cli = TestCli() - cli.setUp() - cli.test_stake() From c983aab39ba16aecd41b98ba30a9f0600b630e9c Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 12 Oct 2022 11:07:04 -0400 Subject: [PATCH 20/27] fix name of n --- bittensor/utils/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 4fde6c6e88..e7d1ea0abe 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -456,8 +456,8 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = hash_rate = 0 # EWMA hash_rate (H/s) - hash_rates = [0] * n # The last n true hash_rates - weights = [alpha_ ** i for i in range(n)] # weights decay by alpha + hash_rates = [0] * n_samples # The last n true hash_rates + weights = [alpha_ ** i for i in range(n_samples)] # weights decay by alpha while not wallet.is_registered(subtensor): # Wait until a solver finds a solution @@ -584,7 +584,7 @@ def __exit__(self, *args): multiprocessing.set_start_method(self._old_start_method, force=True) -def solve_for_difficulty_fast_cuda( subtensor: 'bittensor.Subtensor', wallet: 'bittensor.Wallet', update_interval: int = 50_000, TPB: int = 512, dev_id: Union[List[int], int] = 0, n: int = 5, alpha_: float = 0.70 ) -> Optional[POWSolution]: +def solve_for_difficulty_fast_cuda( subtensor: 'bittensor.Subtensor', wallet: 'bittensor.Wallet', update_interval: int = 50_000, TPB: int = 512, dev_id: Union[List[int], int] = 0, n_samples: int = 5, alpha_: float = 0.70 ) -> Optional[POWSolution]: """ Solves the registration fast using CUDA Args: @@ -676,8 +676,8 @@ def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: mu solution = None hash_rate = 0 # EWMA hash_rate (H/s) - hash_rates = [0] * n # The last n true hash_rates - weights = [alpha_ ** i for i in range(n)] # weights decay by alpha + hash_rates = [0] * n_samples # The last n true hash_rates + weights = [alpha_ ** i for i in range(n_samples)] # weights decay by alpha while not wallet.is_registered(subtensor): # Wait until a solver finds a solution From 4579ba9fdd8be086f623fd5d2ff078a8a8faa750 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 12 Oct 2022 11:15:09 -0400 Subject: [PATCH 21/27] [BIT 584] [feature] btcli register output stats not in place (#923) * add flags for output_in_place during registration * stop tracking best * refactor registration logging output * fix reregister from type bool * change in_place and use_cuda to strtobool * add param and defaults * fix reference before assignment * add new logger to cuda rege * pass param to btcli register call * oops * fix init * try slight timeout * try fix * oop * ? * fix use_cuda flag * add test for new use_cuda flag setup * use create pow to patch * all no prompt dev id * fix console.error * use lower for str comparison * call self register instead * add test for wallet register call * tests are for wallet reregister * fix typo * no self on top-level test * fix tests? * use reregister * typo in test * fix assert * fix assert * should be False * fix time output to use timedelta * add log verbose as option to reg output * should be action * fix typo * add missing function arg * fix spacing * fix flags * fix flags * fix test * should pass in args to config pre-parse * use None instead of NA Co-authored-by: isabella618033 <49876827+isabella618033@users.noreply.github.com> Co-authored-by: Unconst <32490803+unconst@users.noreply.github.com> --- bittensor/_cli/__init__.py | 58 ++-- bittensor/_cli/cli_impl.py | 6 +- bittensor/_config/__init__.py | 8 +- bittensor/_subtensor/__init__.py | 22 +- bittensor/_subtensor/subtensor_impl.py | 8 +- bittensor/_wallet/__init__.py | 3 +- bittensor/_wallet/wallet_impl.py | 18 +- bittensor/utils/__init__.py | 284 +++++++++++++----- tests/integration_tests/test_cli.py | 85 ++++-- .../unit_tests/bittensor_tests/test_wallet.py | 130 +++++++- 10 files changed, 474 insertions(+), 148 deletions(-) diff --git a/bittensor/_cli/__init__.py b/bittensor/_cli/__init__.py index a69a65b65f..eb7c1fd374 100644 --- a/bittensor/_cli/__init__.py +++ b/bittensor/_cli/__init__.py @@ -832,32 +832,38 @@ def check_overview_config( config: 'bittensor.Config' ): def _check_for_cuda_reg_config( config: 'bittensor.Config' ) -> None: """Checks, when CUDA is available, if the user would like to register with their CUDA device.""" if torch.cuda.is_available(): - if config.subtensor.register.cuda.get('use_cuda') is None: - # Ask about cuda registration only if a CUDA device is available. - cuda = Confirm.ask("Detected CUDA device, use CUDA for registration?\n") - config.subtensor.register.cuda.use_cuda = cuda - - # Only ask about which CUDA device if the user has more than one CUDA device. - if config.subtensor.register.cuda.use_cuda and config.subtensor.register.cuda.get('dev_id') is None and torch.cuda.device_count() > 0: - devices: List[str] = [str(x) for x in range(torch.cuda.device_count())] - device_names: List[str] = [torch.cuda.get_device_name(x) for x in range(torch.cuda.device_count())] - console.print("Available CUDA devices:") - choices_str: str = "" - for i, device in enumerate(devices): - choices_str += (" {}: {}\n".format(device, device_names[i])) - console.print(choices_str) - dev_id = IntListPrompt.ask("Which GPU(s) would you like to use? Please list one, or comma-separated", choices=devices, default='All') - if dev_id == 'All': - dev_id = list(range(torch.cuda.device_count())) - else: - try: - # replace the commas with spaces then split over whitespace., - # then strip the whitespace and convert to ints. - dev_id = [int(dev_id.strip()) for dev_id in dev_id.replace(',', ' ').split()] - except ValueError: - console.error(":cross_mark:[red]Invalid GPU device[/red] [bold white]{}[/bold white]\nAvailable CUDA devices:{}".format(dev_id, choices_str)) - sys.exit(1) - config.subtensor.register.cuda.dev_id = dev_id + if not config.no_prompt: + if config.subtensor.register.cuda.get('use_cuda') == None: # flag not set + # Ask about cuda registration only if a CUDA device is available. + cuda = Confirm.ask("Detected CUDA device, use CUDA for registration?\n") + config.subtensor.register.cuda.use_cuda = cuda + + + # Only ask about which CUDA device if the user has more than one CUDA device. + if config.subtensor.register.cuda.use_cuda and config.subtensor.register.cuda.get('dev_id') is None: + devices: List[str] = [str(x) for x in range(torch.cuda.device_count())] + device_names: List[str] = [torch.cuda.get_device_name(x) for x in range(torch.cuda.device_count())] + console.print("Available CUDA devices:") + choices_str: str = "" + for i, device in enumerate(devices): + choices_str += (" {}: {}\n".format(device, device_names[i])) + console.print(choices_str) + dev_id = IntListPrompt.ask("Which GPU(s) would you like to use? Please list one, or comma-separated", choices=devices, default='All') + if dev_id.lower() == 'all': + dev_id = list(range(torch.cuda.device_count())) + else: + try: + # replace the commas with spaces then split over whitespace., + # then strip the whitespace and convert to ints. + dev_id = [int(dev_id.strip()) for dev_id in dev_id.replace(',', ' ').split()] + except ValueError: + console.log(":cross_mark:[red]Invalid GPU device[/red] [bold white]{}[/bold white]\nAvailable CUDA devices:{}".format(dev_id, choices_str)) + sys.exit(1) + config.subtensor.register.cuda.dev_id = dev_id + else: + # flag was not set, use default value. + if config.subtensor.register.cuda.get('use_cuda') is None: + config.subtensor.register.cuda.use_cuda = bittensor.defaults.subtensor.register.cuda.use_cuda def check_register_config( config: 'bittensor.Config' ): if config.subtensor.get('network') == bittensor.defaults.subtensor.network and not config.no_prompt: diff --git a/bittensor/_cli/cli_impl.py b/bittensor/_cli/cli_impl.py index 369da862be..de117d9a4e 100644 --- a/bittensor/_cli/cli_impl.py +++ b/bittensor/_cli/cli_impl.py @@ -246,8 +246,10 @@ def register( self ): TPB = self.config.subtensor.register.cuda.get('TPB', None), update_interval = self.config.subtensor.register.get('update_interval', None), num_processes = self.config.subtensor.register.get('num_processes', None), - cuda = self.config.subtensor.register.cuda.get('use_cuda', None), - dev_id = self.config.subtensor.register.cuda.get('dev_id', None) + cuda = self.config.subtensor.register.cuda.get('use_cuda', bittensor.defaults.subtensor.register.cuda.use_cuda), + dev_id = self.config.subtensor.register.cuda.get('dev_id', None), + output_in_place = self.config.subtensor.register.get('output_in_place', bittensor.defaults.subtensor.register.output_in_place), + log_verbose = self.config.subtensor.register.get('verbose', bittensor.defaults.subtensor.register.verbose), ) def transfer( self ): diff --git a/bittensor/_config/__init__.py b/bittensor/_config/__init__.py index 76b4eff293..a327ca451c 100644 --- a/bittensor/_config/__init__.py +++ b/bittensor/_config/__init__.py @@ -68,16 +68,16 @@ def __new__( cls, parser: ArgumentParser = None, strict: bool = False, args: Opt # this can fail if the --config has already been added. pass + # Get args from argv if not passed in. + if args == None: + args = sys.argv[1:] + # 1.1 Optionally load defaults if the --config is set. try: config_file_path = str(os.getcwd()) + '/' + vars(parser.parse_known_args(args)[0])['config'] except Exception as e: config_file_path = None - # Get args from argv if not passed in. - if args == None: - args = sys.argv[1:] - # Parse args not strict params = cls.__parse_args__(args=args, parser=parser, strict=False) diff --git a/bittensor/_subtensor/__init__.py b/bittensor/_subtensor/__init__.py index 8dd68c973a..eab11fd1cc 100644 --- a/bittensor/_subtensor/__init__.py +++ b/bittensor/_subtensor/__init__.py @@ -23,6 +23,8 @@ from substrateinterface import SubstrateInterface from torch.cuda import is_available as is_cuda_available +from bittensor.utils import strtobool_with_default + from . import subtensor_impl, subtensor_mock logger = logger.opt(colors=True) @@ -187,13 +189,17 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ): help='''The subtensor endpoint flag. If set, overrides the --network flag. ''') parser.add_argument('--' + prefix_str + 'subtensor._mock', action='store_true', help='To turn on subtensor mocking for testing purposes.', default=bittensor.defaults.subtensor._mock) - - parser.add_argument('--' + prefix_str + 'subtensor.register.num_processes', '-n', dest='subtensor.register.num_processes', help="Number of processors to use for registration", type=int, default=bittensor.defaults.subtensor.register.num_processes) + # registration args. Used for register and re-register and anything that calls register. + parser.add_argument('--' + prefix_str + 'subtensor.register.num_processes', '-n', dest=prefix_str + 'subtensor.register.num_processes', help="Number of processors to use for registration", type=int, default=bittensor.defaults.subtensor.register.num_processes) parser.add_argument('--' + prefix_str + 'subtensor.register.update_interval', '--' + prefix_str + 'subtensor.register.cuda.update_interval', '--' + prefix_str + 'cuda.update_interval', '-u', help="The number of nonces to process before checking for next block during registration", type=int, default=bittensor.defaults.subtensor.register.update_interval) - # registration args. Used for register and re-register and anything that calls register. - parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.use_cuda', '--' + prefix_str + 'cuda', '--' + prefix_str + 'cuda.use_cuda', default=argparse.SUPPRESS, help='''Set true to use CUDA.''', action='store_true', required=False ) - parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.dev_id', '--' + prefix_str + 'cuda.dev_id', type=int, nargs='+', default=argparse.SUPPRESS, help='''Set the CUDA device id(s). Goes by the order of speed. (i.e. 0 is the fastest).''', required=False ) + parser.add_argument('--' + prefix_str + 'subtensor.register.output_in_place', help="Whether to ouput the registration statistics in-place. Set flag to enable.", action='store_true', required=False, default=bittensor.defaults.subtensor.register.output_in_place) + parser.add_argument('--' + prefix_str + 'subtensor.register.verbose', help="Whether to ouput the registration statistics verbosely.", action='store_true', required=False, default=bittensor.defaults.subtensor.register.verbose) + + ## Registration args for CUDA registration. + parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.use_cuda', '--' + prefix_str + 'cuda', '--' + prefix_str + 'cuda.use_cuda', default=argparse.SUPPRESS, help='''Set flag to use CUDA to register.''', action="store_true", required=False ) + parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.no_cuda', '--' + prefix_str + 'no_cuda', '--' + prefix_str + 'cuda.no_cuda', dest=prefix_str + 'subtensor.register.cuda.use_cuda', default=argparse.SUPPRESS, help='''Set flag to not use CUDA for registration''', action="store_false", required=False ) + parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.dev_id', '--' + prefix_str + 'cuda.dev_id', type=int, nargs='+', default=argparse.SUPPRESS, help='''Set the CUDA device id(s). Goes by the order of speed. (i.e. 0 is the fastest).''', required=False ) parser.add_argument( '--' + prefix_str + 'subtensor.register.cuda.TPB', '--' + prefix_str + 'cuda.TPB', type=int, default=bittensor.defaults.subtensor.register.cuda.TPB, help='''Set the number of Threads Per Block for CUDA.''', required=False ) except argparse.ArgumentError: @@ -212,12 +218,16 @@ def add_defaults(cls, defaults ): defaults.subtensor.register = bittensor.Config() defaults.subtensor.register.num_processes = os.getenv('BT_SUBTENSOR_REGISTER_NUM_PROCESSES') if os.getenv('BT_SUBTENSOR_REGISTER_NUM_PROCESSES') != None else None # uses processor count by default within the function defaults.subtensor.register.update_interval = os.getenv('BT_SUBTENSOR_REGISTER_UPDATE_INTERVAL') if os.getenv('BT_SUBTENSOR_REGISTER_UPDATE_INTERVAL') != None else 50_000 + defaults.subtensor.register.output_in_place = True + defaults.subtensor.register.verbose = False defaults.subtensor.register.cuda = bittensor.Config() defaults.subtensor.register.cuda.dev_id = [0] defaults.subtensor.register.cuda.use_cuda = False defaults.subtensor.register.cuda.TPB = 256 + + @staticmethod def check_config( config: 'bittensor.Config' ): assert config.subtensor @@ -225,7 +235,7 @@ def check_config( config: 'bittensor.Config' ): if config.subtensor.get('register') and config.subtensor.register.get('cuda'): assert all((isinstance(x, int) or isinstance(x, str) and x.isnumeric() ) for x in config.subtensor.register.cuda.get('dev_id', [])) - if config.subtensor.register.cuda.get('use_cuda', False): + if config.subtensor.register.cuda.get('use_cuda', bittensor.defaults.subtensor.register.cuda.use_cuda): try: import cubit except ImportError: diff --git a/bittensor/_subtensor/subtensor_impl.py b/bittensor/_subtensor/subtensor_impl.py index 9770c1d001..747826c59b 100644 --- a/bittensor/_subtensor/subtensor_impl.py +++ b/bittensor/_subtensor/subtensor_impl.py @@ -500,11 +500,13 @@ def register ( wait_for_finalization: bool = True, prompt: bool = False, max_allowed_attempts: int = 3, + output_in_place: bool = True, cuda: bool = False, dev_id: Union[List[int], int] = 0, TPB: int = 256, num_processes: Optional[int] = None, update_interval: Optional[int] = None, + log_verbose: bool = False, ) -> bool: r""" Registers the wallet to chain. Args: @@ -530,6 +532,8 @@ def register ( The number of processes to use to register. update_interval (int): The number of nonces to solve between updates. + log_verbose (bool): + If true, the registration process will log more information. Returns: success (bool): flag is true if extrinsic was finalized or uncluded in the block. @@ -556,9 +560,9 @@ def register ( if prompt: bittensor.__console__.error('CUDA is not available.') return False - pow_result = bittensor.utils.create_pow( self, wallet, cuda, dev_id, TPB, num_processes=num_processes, update_interval=update_interval ) + pow_result = bittensor.utils.create_pow( self, wallet, output_in_place, cuda, dev_id, TPB, num_processes=num_processes, update_interval=update_interval, log_verbose=log_verbose ) else: - pow_result = bittensor.utils.create_pow( self, wallet, num_processes=num_processes, update_interval=update_interval) + pow_result = bittensor.utils.create_pow( self, wallet, output_in_place, num_processes=num_processes, update_interval=update_interval, log_verbose=log_verbose ) # pow failed if not pow_result: diff --git a/bittensor/_wallet/__init__.py b/bittensor/_wallet/__init__.py index 3f83f6b40d..090b7c3054 100644 --- a/bittensor/_wallet/__init__.py +++ b/bittensor/_wallet/__init__.py @@ -23,6 +23,7 @@ import os import bittensor +from bittensor.utils import strtobool from . import wallet_impl, wallet_mock @@ -115,7 +116,7 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ): parser.add_argument('--' + prefix_str + 'wallet.hotkeys', '--' + prefix_str + 'wallet.exclude_hotkeys', required=False, action='store', default=bittensor.defaults.wallet.hotkeys, type=str, nargs='*', help='''Specify the hotkeys by name. (e.g. hk1 hk2 hk3)''') parser.add_argument('--' + prefix_str + 'wallet.all_hotkeys', required=False, action='store_true', default=bittensor.defaults.wallet.all_hotkeys, help='''To specify all hotkeys. Specifying hotkeys will exclude them from this all.''') - parser.add_argument('--' + prefix_str + 'wallet.reregister', required=False, default=bittensor.defaults.wallet.reregister, type=lambda x: bool(strtobool(x)), help='''Whether to reregister the wallet if it is not already registered.''') + parser.add_argument('--' + prefix_str + 'wallet.reregister', required=False, action='store', default=bittensor.defaults.wallet.reregister, type=strtobool, help='''Whether to reregister the wallet if it is not already registered.''') except argparse.ArgumentError as e: pass diff --git a/bittensor/_wallet/wallet_impl.py b/bittensor/_wallet/wallet_impl.py index 5749c487ce..a02cc1319c 100644 --- a/bittensor/_wallet/wallet_impl.py +++ b/bittensor/_wallet/wallet_impl.py @@ -246,16 +246,18 @@ def reregister( if not self.config.wallet.get('reregister'): sys.exit(0) - subtensor.register( - wallet = self, + self.register( + subtensor = subtensor, prompt = prompt, TPB = self.config.subtensor.register.cuda.get('TPB', None), update_interval = self.config.subtensor.register.cuda.get('update_interval', None), num_processes = self.config.subtensor.register.get('num_processes', None), - cuda = self.config.subtensor.register.cuda.get('use_cuda', None), + cuda = self.config.subtensor.register.cuda.get('use_cuda', bittensor.defaults.subtensor.register.cuda.use_cuda), dev_id = self.config.subtensor.register.cuda.get('dev_id', None), wait_for_inclusion = wait_for_inclusion, wait_for_finalization = wait_for_finalization, + output_in_place = self.config.subtensor.register.get('output_in_place', bittensor.defaults.subtensor.register.output_in_place), + log_verbose = self.config.subtensor.register.get('verbose', bittensor.defaults.subtensor.register.verbose), ) return self @@ -272,6 +274,8 @@ def register ( TPB: int = 256, num_processes: Optional[int] = None, update_interval: Optional[int] = None, + output_in_place: bool = True, + log_verbose: bool = False, ) -> 'bittensor.Wallet': """ Registers the wallet to chain. Args: @@ -297,6 +301,10 @@ def register ( The number of processes to use to register. update_interval (int): The number of nonces to solve between updates. + output_in_place (bool): + If true, the registration output is printed in-place. + log_verbose (bool): + If true, the registration output is more verbose. Returns: success (bool): flag is true if extrinsic was finalized or uncluded in the block. @@ -309,11 +317,13 @@ def register ( wait_for_inclusion = wait_for_inclusion, wait_for_finalization = wait_for_finalization, prompt=prompt, max_allowed_attempts=max_allowed_attempts, + output_in_place = output_in_place, cuda=cuda, dev_id=dev_id, TPB=TPB, num_processes=num_processes, - update_interval=update_interval + update_interval=update_interval, + log_verbose=log_verbose, ) return self diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 490dea5c96..ff0d9af119 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -9,7 +9,7 @@ import time from dataclasses import dataclass from queue import Empty -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union, Callable import backoff import bittensor @@ -19,6 +19,8 @@ from Crypto.Hash import keccak from substrateinterface import Keypair from substrateinterface.utils import ss58 +from rich import console as rich_console, status as rich_status +from datetime import timedelta from .register_cuda import solve_cuda @@ -156,9 +158,6 @@ class SolverBase(multiprocessing.Process): The total number of processes running. update_interval: int The number of nonces to try to solve before checking for a new block. - best_queue: multiprocessing.Queue - The queue to put the best nonce the process has found during the pow solve. - New nonces are added each update_interval. time_queue: multiprocessing.Queue The queue to put the time the process took to finish each update_interval. Used for calculating the average time per update_interval across all processes. @@ -193,7 +192,6 @@ class SolverBase(multiprocessing.Process): proc_num: int num_proc: int update_interval: int - best_queue: Optional[multiprocessing.Queue] time_queue: multiprocessing.Queue solution_queue: multiprocessing.Queue newBlockEvent: multiprocessing.Event @@ -204,12 +202,11 @@ class SolverBase(multiprocessing.Process): check_block: multiprocessing.Lock limit: int - def __init__(self, proc_num, num_proc, update_interval, best_queue, time_queue, solution_queue, stopEvent, curr_block, curr_block_num, curr_diff, check_block, limit): + def __init__(self, proc_num, num_proc, update_interval, time_queue, solution_queue, stopEvent, curr_block, curr_block_num, curr_diff, check_block, limit): multiprocessing.Process.__init__(self) self.proc_num = proc_num self.num_proc = num_proc self.update_interval = update_interval - self.best_queue = best_queue self.time_queue = time_queue self.solution_queue = solution_queue self.newBlockEvent = multiprocessing.Event() @@ -264,7 +261,7 @@ class CUDASolver(SolverBase): TPB: int def __init__(self, proc_num, num_proc, update_interval, time_queue, solution_queue, stopEvent, curr_block, curr_block_num, curr_diff, check_block, limit, dev_id: int, TPB: int): - super().__init__(proc_num, num_proc, update_interval, None, time_queue, solution_queue, stopEvent, curr_block, curr_block_num, curr_diff, check_block, limit) + super().__init__(proc_num, num_proc, update_interval, time_queue, solution_queue, stopEvent, curr_block, curr_block_num, curr_diff, check_block, limit) self.dev_id = dev_id self.TPB = TPB @@ -327,8 +324,6 @@ def solve_for_nonce_block_cuda(solver: CUDASolver, nonce_start: int, update_inte def solve_for_nonce_block(solver: Solver, nonce_start: int, nonce_end: int, block_bytes: bytes, difficulty: int, limit: int, block_number: int) -> Tuple[Optional[POWSolution], int]: - best_local = float('inf') - best_seal_local = [0]*32 start = time.time() for nonce in range(nonce_start, nonce_end): # Create seal. @@ -345,12 +340,6 @@ def solve_for_nonce_block(solver: Solver, nonce_start: int, nonce_end: int, bloc # Found a solution, save it. return POWSolution(nonce, block_number, difficulty, seal), time.time() - start - if (product - limit) < best_local: - best_local = product - limit - best_seal_local = seal - - # Send best solution to best queue. - solver.best_queue.put((best_local, best_seal_local)) return None, time.time() - start @@ -372,6 +361,7 @@ def update_curr_block(curr_diff: multiprocessing.Array, curr_block: multiprocess curr_block[i] = block_bytes[i] registration_diff_pack(diff, curr_diff) + def get_cpu_count(): try: return len(os.sched_getaffinity(0)) @@ -379,7 +369,65 @@ def get_cpu_count(): # OSX does not have sched_getaffinity return os.cpu_count() -def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = None, update_interval: Optional[int] = None ) -> Optional[POWSolution]: +@dataclass +class RegistrationStatistics: + """Statistics for a registration.""" + time_spent_total: float + time_average_perpetual: float + rounds_total: int + time_average: float + time_spent: float + hash_rate_perpetual: float + hash_rate: float + difficulty: int + block_number: int + block_hash: bytes + + +class RegistrationStatisticsLogger: + """Logs statistics for a registration.""" + console: rich_console.Console + status: Optional[rich_status.Status] + + def __init__( self, console: rich_console.Console, output_in_place: bool = True) -> None: + self.console = console + + if output_in_place: + self.status = self.console.status("Solving") + else: + self.status = None + + def start( self ) -> None: + if self.status is not None: + self.status.start() + + def stop( self ) -> None: + if self.status is not None: + self.status.stop() + + + def get_status_message(cls, stats: RegistrationStatistics, verbose: bool = False) -> str: + message = f"""Solving + time spent: {timedelta(seconds=stats.time_spent)}""" + \ + (f""" + time spent total: {stats.time_spent_total:.2f} s + time average perpetual: {timedelta(seconds=stats.time_average_perpetual)} + """ if verbose else "") + f""" + Difficulty: [bold white]{millify(stats.difficulty)}[/bold white] + Iters: [bold white]{get_human_readable(int(stats.hash_rate), 'H')}/s[/bold white] + Block: [bold white]{stats.block_number}[/bold white] + Block_hash: [bold white]{stats.block_hash.encode('utf-8')}[/bold white]""" + return message.replace(" ", "") + + + def update( self, stats: RegistrationStatistics, verbose: bool = False ) -> None: + if self.status is not None: + self.status.update( self.get_status_message(stats, verbose=verbose) ) + else: + self.console.log( self.get_status_message(stats, verbose=verbose), ) + + +def solve_for_difficulty_fast( subtensor, wallet, output_in_place: bool = True, num_processes: Optional[int] = None, update_interval: Optional[int] = None, log_verbose: bool = False ) -> Optional[POWSolution]: """ Solves the POW for registration using multiprocessing. Args: @@ -387,10 +435,14 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = Subtensor to connect to for block information and to submit. wallet: Wallet to use for registration. + output_in_place: bool + If true, prints the status in place. Otherwise, prints the status on a new line. num_processes: int Number of processes to use. update_interval: int Number of nonces to solve before updating block information. + log_verbose: bool + If true, prints more verbose logging of the registration metrics. Note: - We can also modify the update interval to do smaller blocks of work, while still updating the block information after a different number of nonces, @@ -405,30 +457,21 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = limit = int(math.pow(2,256)) - 1 - console = bittensor.__console__ - status = console.status("Solving") - - best_seal: bytes - best_number: int - best_number = float('inf') - curr_block = multiprocessing.Array('h', 64, lock=True) # byte array curr_block_num = multiprocessing.Value('i', 0, lock=True) # int curr_diff = multiprocessing.Array('Q', [0, 0], lock=True) # [high, low] - - status.start() # Establish communication queues ## See the Solver class for more information on the queues. stopEvent = multiprocessing.Event() stopEvent.clear() - best_queue = multiprocessing.Queue() + solution_queue = multiprocessing.Queue() time_queue = multiprocessing.Queue() check_block = multiprocessing.Lock() # Start consumers - solvers = [ Solver(i, num_processes, update_interval, best_queue, time_queue, solution_queue, stopEvent, curr_block, curr_block_num, curr_diff, check_block, limit) + solvers = [ Solver(i, num_processes, update_interval, time_queue, solution_queue, stopEvent, curr_block, curr_block_num, curr_diff, check_block, limit) for i in range(num_processes) ] # Get first block @@ -449,11 +492,30 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = for w in solvers: w.start() # start the solver processes - start_time = time.time() + curr_stats = RegistrationStatistics( + time_spent_total = 0.0, + time_average_perpetual = 0.0, + time_average = 0.0, + rounds_total = 0, + time_spent = 0.0, + hash_rate_perpetual = 0.0, + hash_rate = 0.0, + difficulty = difficulty, + block_number = block_number, + block_hash = block_hash + ) + + start_time_perpetual = time.time() + + console = bittensor.__console__ + logger = RegistrationStatisticsLogger(console, output_in_place) + logger.start() + solution = None - best_seal = None - itrs_per_sec = 0 + while not wallet.is_registered(subtensor): + start_time = time.time() + time_avg: Optional[float] = None # Wait until a solver finds a solution try: solution = solution_queue.get(block=True, timeout=0.25) @@ -478,6 +540,11 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = # Set new block events for each solver for w in solvers: w.newBlockEvent.set() + + # update stats + curr_stats.block_number = block_number + curr_stats.block_hash = block_hash + curr_stats.difficulty = difficulty # Get times for each solver time_total = 0 @@ -493,31 +560,22 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: Optional[int] = # Calculate average time per solver for the update_interval if num_time > 0: time_avg = time_total / num_time - itrs_per_sec = update_interval*num_processes / time_avg - - # get best solution from each solver using the best_queue - for _ in solvers: - try: - num, seal = best_queue.get_nowait() - if num < best_number: - best_number = num - best_seal = seal - - except Empty: - break + curr_stats.hash_rate = update_interval*num_processes / time_avg - message = f"""Solving - time spent: {time.time() - start_time} - Difficulty: [bold white]{millify(difficulty)}[/bold white] - Iters: [bold white]{get_human_readable(int(itrs_per_sec), 'H')}/s[/bold white] - Block: [bold white]{block_number}[/bold white] - Block_hash: [bold white]{block_hash.encode('utf-8')}[/bold white] - Best: [bold white]{binascii.hexlify(bytes(best_seal) if best_seal else bytes(0))}[/bold white]""" - status.update(message.replace(" ", "")) + curr_stats.time_spent = time.time() - start_time + new_time_spent_total = time.time() - start_time_perpetual + curr_stats.time_average = time_avg if not None else curr_stats.time_average + curr_stats.time_average_perpetual = (curr_stats.time_average_perpetual*curr_stats.rounds_total + curr_stats.time_spent)/(curr_stats.rounds_total+1) + curr_stats.rounds_total += 1 + curr_stats.hash_rate_perpetual = (curr_stats.time_spent_total*curr_stats.hash_rate_perpetual + curr_stats.hash_rate)/ new_time_spent_total + curr_stats.time_spent_total = new_time_spent_total + + # Update the logger + logger.update(curr_stats, verbose=log_verbose) # exited while, solution contains the nonce or wallet is registered stopEvent.set() # stop all other processes - status.stop() + logger.stop() return solution @@ -565,7 +623,7 @@ def __exit__(self, *args): multiprocessing.set_start_method(self._old_start_method, force=True) -def solve_for_difficulty_fast_cuda( subtensor: 'bittensor.Subtensor', wallet: 'bittensor.Wallet', update_interval: int = 50_000, TPB: int = 512, dev_id: Union[List[int], int] = 0, use_kernel_launch_optimization: bool = False ) -> Optional[POWSolution]: +def solve_for_difficulty_fast_cuda( subtensor: 'bittensor.Subtensor', wallet: 'bittensor.Wallet', output_in_place: bool = True, update_interval: int = 50_000, TPB: int = 512, dev_id: Union[List[int], int] = 0, log_verbose: bool = False ) -> Optional[POWSolution]: """ Solves the registration fast using CUDA Args: @@ -573,12 +631,16 @@ def solve_for_difficulty_fast_cuda( subtensor: 'bittensor.Subtensor', wallet: 'b The subtensor node to grab blocks wallet: bittensor.Wallet The wallet to register + output_in_place: bool + If true, prints the output in place, otherwise prints to new lines update_interval: int The number of nonces to try before checking for more blocks TPB: int The number of threads per block. CUDA param that should match the GPU capability dev_id: Union[List[int], int] The CUDA device IDs to execute the registration on, either a single device or a list of devices + log_verbose: bool + If true, prints more verbose logging of the registration metrics. """ if isinstance(dev_id, int): dev_id = [dev_id] @@ -593,11 +655,7 @@ def solve_for_difficulty_fast_cuda( subtensor: 'bittensor.Subtensor', wallet: 'b limit = int(math.pow(2,256)) - 1 - console = bittensor.__console__ - status = console.status("Solving") - # Set mp start to use spawn so CUDA doesn't complain - # Force the set start method in-case of re-register with UsingSpawnStartMethod(force=True): curr_block = multiprocessing.Array('h', 64, lock=True) # byte array curr_block_num = multiprocessing.Value('i', 0, lock=True) # int @@ -610,8 +668,6 @@ def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: mu curr_block[i] = block_bytes[i] registration_diff_pack(diff, curr_diff) - status.start() - # Establish communication queues stopEvent = multiprocessing.Event() stopEvent.clear() @@ -633,6 +689,7 @@ def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: mu block_hash = subtensor.substrate.get_block_hash( block_number ) block_bytes = block_hash.encode('utf-8')[2:] old_block_number = block_number + # Set to current block update_curr_block(block_number, block_bytes, difficulty, check_block) @@ -643,11 +700,30 @@ def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: mu for w in solvers: w.start() # start the solver processes - start_time = time.time() - time_since = 0.0 + curr_stats = RegistrationStatistics( + time_spent_total = 0.0, + time_average_perpetual = 0.0, + time_average = 0.0, + rounds_total = 0, + time_spent = 0.0, + hash_rate_perpetual = 0.0, + hash_rate = 0.0, + difficulty = difficulty, + block_number = block_number, + block_hash = block_hash + ) + + start_time_perpetual = time.time() + + console = bittensor.__console__ + logger = RegistrationStatisticsLogger(console, output_in_place) + logger.start() + solution = None - itrs_per_sec = 0 + while not wallet.is_registered(subtensor): + start_time = time.time() + time_avg: Optional[float] = None # Wait until a solver finds a solution try: solution = solution_queue.get(block=True, timeout=0.15) @@ -657,8 +733,6 @@ def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: mu # No solution found, try again pass - # check for new block - block_number = subtensor.get_current_block() if block_number != old_block_number: old_block_number = block_number # update block information @@ -672,13 +746,18 @@ def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: mu # Set new block events for each solver for w in solvers: w.newBlockEvent.set() + + # update stats + curr_stats.block_number = block_number + curr_stats.block_hash = block_hash + curr_stats.difficulty = difficulty # Get times for each solver time_total = 0 num_time = 0 for _ in solvers: try: - time_ = time_queue.get_nowait() + time_ = time_queue.get(timeout=0.01) time_total += time_ num_time += 1 @@ -687,32 +766,48 @@ def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: mu if num_time > 0: time_avg = time_total / num_time - itrs_per_sec = TPB*update_interval*num_processes / time_avg - time_since = time.time() - start_time + curr_stats.hash_rate = TPB*update_interval*num_processes / time_avg - message = f"""Solving - time spent: {time_since} - Difficulty: [bold white]{millify(difficulty)}[/bold white] - Iters: [bold white]{get_human_readable(int(itrs_per_sec), 'H')}/s[/bold white] - Block: [bold white]{block_number}[/bold white] - Block_hash: [bold white]{block_hash.encode('utf-8')}[/bold white]""" - status.update(message.replace(" ", "")) + curr_stats.time_spent = time.time() - start_time + new_time_spent_total = time.time() - start_time_perpetual + curr_stats.time_average = time_avg if not None else curr_stats.time_average + curr_stats.time_average_perpetual = (curr_stats.time_average_perpetual*curr_stats.rounds_total + curr_stats.time_spent)/(curr_stats.rounds_total+1) + curr_stats.rounds_total += 1 + curr_stats.hash_rate_perpetual = (curr_stats.time_spent_total*curr_stats.hash_rate_perpetual + curr_stats.hash_rate)/ new_time_spent_total + curr_stats.time_spent_total = new_time_spent_total + + # Update the logger + logger.update(curr_stats, verbose=log_verbose) # exited while, found_solution contains the nonce or wallet is registered if solution is not None: stopEvent.set() # stop all other processes - status.stop() + logger.stop() return solution - status.stop() + logger.stop() return None -def create_pow( subtensor, wallet, cuda: bool = False, dev_id: Union[List[int], int] = 0, tpb: int = 256, num_processes: int = None, update_interval: int = None) -> Optional[Dict[str, Any]]: +def create_pow( + subtensor, + wallet, + output_in_place: bool = True, + cuda: bool = False, + dev_id: Union[List[int], int] = 0, + tpb: int = 256, + num_processes: int = None, + update_interval: int = None, + log_verbose: bool = False + ) -> Optional[Dict[str, Any]]: if cuda: - solution: POWSolution = solve_for_difficulty_fast_cuda( subtensor, wallet, dev_id=dev_id, TPB=tpb, update_interval=update_interval ) + solution: POWSolution = solve_for_difficulty_fast_cuda( subtensor, wallet, output_in_place=output_in_place, \ + dev_id=dev_id, TPB=tpb, update_interval=update_interval, log_verbose=log_verbose + ) else: - solution: POWSolution = solve_for_difficulty_fast( subtensor, wallet, num_processes=num_processes, update_interval=update_interval ) + solution: POWSolution = solve_for_difficulty_fast( subtensor, wallet, output_in_place=output_in_place, \ + num_processes=num_processes, update_interval=update_interval, log_verbose=log_verbose + ) return None if solution is None else { 'nonce': solution.nonce, @@ -800,3 +895,34 @@ def is_valid_bittensor_address_or_public_key( address: Union[str, bytes] ) -> bo else: # Invalid address type return False + +def strtobool_with_default( default: bool ) -> Callable[[str], bool]: + """ + Creates a strtobool function with a default value. + + Args: + default(bool): The default value to return if the string is empty. + + Returns: + The strtobool function with the default value. + """ + return lambda x: strtobool(x) if x != "" else default + + +def strtobool(val: str) -> bool: + """ + Converts a string to a boolean value. + + truth-y values are 'y', 'yes', 't', 'true', 'on', and '1'; + false-y values are 'n', 'no', 'f', 'false', 'off', and '0'. + + Raises ValueError if 'val' is anything else. + """ + val = val.lower() + if val in ('y', 'yes', 't', 'true', 'on', '1'): + return True + elif val in ('n', 'no', 'f', 'false', 'off', '0'): + return False + else: + raise ValueError("invalid truth value %r" % (val,)) + diff --git a/tests/integration_tests/test_cli.py b/tests/integration_tests/test_cli.py index 4b8a7985d4..73ad97227e 100644 --- a/tests/integration_tests/test_cli.py +++ b/tests/integration_tests/test_cli.py @@ -1086,7 +1086,7 @@ def test_register( self ): with patch('bittensor.Subtensor.register', return_value=True): cli = bittensor.cli(config) cli.run() - + def test_stake( self ): wallet = TestCli.generate_wallet() bittensor.Subtensor.neuron_for_pubkey = MagicMock(return_value=self.mock_neuron) @@ -1327,33 +1327,72 @@ def test_list_no_wallet( self ): # This shouldn't raise an error anymore cli.run() -def test_btcli_help(): - """ - Verify the correct help text is output when the --help flag is passed - """ - with pytest.raises(SystemExit) as pytest_wrapped_e: - with patch('argparse.ArgumentParser._print_message', return_value=None) as mock_print_message: - args = [ - '--help' + def test_btcli_help(self): + """ + Verify the correct help text is output when the --help flag is passed + """ + with pytest.raises(SystemExit) as pytest_wrapped_e: + with patch('argparse.ArgumentParser._print_message', return_value=None) as mock_print_message: + args = [ + '--help' + ] + bittensor.cli(args=args).run() + + # Should try to print help + mock_print_message.assert_called_once() + + call_args = mock_print_message.call_args + args, _ = call_args + help_out = args[0] + + # Expected help output even if parser isn't working well + ## py3.6-3.9 or py3.10+ + assert 'optional arguments' in help_out or 'options' in help_out + # Expected help output if all commands are listed + assert 'positional arguments' in help_out + # Verify that cli is printing the help message for + assert 'overview' in help_out + assert 'run' in help_out + + + def test_register_cuda_use_cuda_flag(self): + class ExitEarlyException(Exception): + """Raised by mocked function to exit early""" + pass + + base_args = [ + "register", + "--subtensor._mock", + "--subtensor.network", "mock", + "--wallet.path", "tmp/walletpath", + "--wallet.name", "mock", + "--wallet.hotkey", "hk0", + "--no_prompt", + "--cuda.dev_id", "0", ] - bittensor.cli(args=args).run() - # Should try to print help - mock_print_message.assert_called_once() + with patch('torch.cuda.is_available', return_value=True): + with patch('bittensor.Subtensor.register', side_effect=ExitEarlyException): + # Should be able to set true without argument + args = base_args + [ + "--subtensor.register.cuda.use_cuda", # should be True without any arugment + ] + with pytest.raises(ExitEarlyException): + cli = bittensor.cli(args=args) + cli.run() + + assert cli.config.subtensor.register.cuda.get('use_cuda') == True # should be None - call_args = mock_print_message.call_args - args, _ = call_args - help_out = args[0] + # Should be able to set to false with no argument - # Expected help output even if parser isn't working well - ## py3.6-3.9 or py3.10+ - assert 'optional arguments' in help_out or 'options' in help_out - # Expected help output if all commands are listed - assert 'positional arguments' in help_out - # Verify that cli is printing the help message for - assert 'overview' in help_out - assert 'run' in help_out + args = base_args + [ + "--subtensor.register.cuda.no_cuda", + ] + with pytest.raises(ExitEarlyException): + cli = bittensor.cli(args=args) + cli.run() + assert cli.config.subtensor.register.cuda.use_cuda == False class TestCLIUsingArgs(unittest.TestCase): """ Test the CLI by passing args directly to the bittensor.cli factory diff --git a/tests/unit_tests/bittensor_tests/test_wallet.py b/tests/unit_tests/bittensor_tests/test_wallet.py index 660eb5bf99..2ff6177558 100644 --- a/tests/unit_tests/bittensor_tests/test_wallet.py +++ b/tests/unit_tests/bittensor_tests/test_wallet.py @@ -16,7 +16,7 @@ # DEALINGS IN THE SOFTWARE. import unittest -from unittest.mock import patch +from unittest.mock import patch, MagicMock import pytest import bittensor @@ -94,3 +94,131 @@ def test_regen_hotkey_from_hex_seed_str(self): seed_str_bad = "0x659c024d5be809000d0d93fe378cfde020846150b01c49a201fc2a02041f763" # 1 character short with pytest.raises(ValueError): self.mock_wallet.regenerate_hotkey(seed=seed_str_bad) + +class TestWalletReregister(unittest.TestCase): + def test_wallet_reregister_use_cuda_flag_none(self): + config = bittensor.Config() + config.wallet = bittensor.Config() + config.wallet.reregister = True + + config.subtensor = bittensor.Config() + config.subtensor.register = bittensor.Config() + config.subtensor.register.cuda = bittensor.Config() + config.subtensor.register.cuda.use_cuda = None # don't set the argument, but do specify the flag + # No need to specify the other config options as they are default to None + + mock_wallet = bittensor.wallet.mock() + mock_wallet.config = config + + class MockException(Exception): + pass + + def exit_early(*args, **kwargs): + raise MockException('exit_early') + + with patch('bittensor.Subtensor.register', side_effect=exit_early) as mock_register: + # Should be able to set without argument + with pytest.raises(MockException): + mock_wallet.reregister() + + call_args = mock_register.call_args + _, kwargs = call_args + + mock_register.assert_called_once() + self.assertEqual(kwargs['cuda'], None) # should be None when no argument, but flag set + + def test_wallet_reregister_use_cuda_flag_true(self): + config = bittensor.Config() + config.wallet = bittensor.Config() + config.wallet.reregister = True + + config.subtensor = bittensor.Config() + config.subtensor.register = bittensor.Config() + config.subtensor.register.cuda = bittensor.Config() + config.subtensor.register.cuda.use_cuda = True + config.subtensor.register.cuda.dev_id = 0 + # No need to specify the other config options as they are default to None + + mock_wallet = bittensor.wallet.mock() + mock_wallet.config = config + + class MockException(Exception): + pass + + def exit_early(*args, **kwargs): + raise MockException('exit_early') + + with patch('bittensor.Subtensor.register', side_effect=exit_early) as mock_register: + # Should be able to set without argument + with pytest.raises(MockException): + mock_wallet.reregister() + + call_args = mock_register.call_args + _, kwargs = call_args + + mock_register.assert_called_once() + self.assertEqual(kwargs['cuda'], True) # should be default when no argument + + def test_wallet_reregister_use_cuda_flag_false(self): + config = bittensor.Config() + config.wallet = bittensor.Config() + config.wallet.reregister = True + + config.subtensor = bittensor.Config() + config.subtensor.register = bittensor.Config() + config.subtensor.register.cuda = bittensor.Config() + config.subtensor.register.cuda.use_cuda = False + config.subtensor.register.cuda.dev_id = 0 + # No need to specify the other config options as they are default to None + + mock_wallet = bittensor.wallet.mock() + mock_wallet.config = config + + class MockException(Exception): + pass + + def exit_early(*args, **kwargs): + raise MockException('exit_early') + + with patch('bittensor.Subtensor.register', side_effect=exit_early) as mock_register: + # Should be able to set without argument + with pytest.raises(MockException): + mock_wallet.reregister() + + call_args = mock_register.call_args + _, kwargs = call_args + + mock_register.assert_called_once() + self.assertEqual(kwargs['cuda'], False) # should be default when no argument + + def test_wallet_reregister_use_cuda_flag_not_specified_false(self): + config = bittensor.Config() + config.wallet = bittensor.Config() + config.wallet.reregister = True + + config.subtensor = bittensor.Config() + config.subtensor.register = bittensor.Config() + config.subtensor.register.cuda = bittensor.Config() + #config.subtensor.register.cuda.use_cuda # don't specify the flag + config.subtensor.register.cuda.dev_id = 0 + # No need to specify the other config options as they are default to None + + mock_wallet = bittensor.wallet.mock() + mock_wallet.config = config + + class MockException(Exception): + pass + + def exit_early(*args, **kwargs): + raise MockException('exit_early') + + with patch('bittensor.Subtensor.register', side_effect=exit_early) as mock_register: + # Should be able to set without argument + with pytest.raises(MockException): + mock_wallet.reregister() + + call_args = mock_register.call_args + _, kwargs = call_args + + mock_register.assert_called_once() + self.assertEqual(kwargs['cuda'], False) # should be False when no flag was set From 945b76624020f3a2b553110c29db955215e867d4 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 12 Oct 2022 11:46:09 -0400 Subject: [PATCH 22/27] fix verbose log --- bittensor/utils/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 1e390ba2ba..7ae9425f5e 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -406,8 +406,8 @@ def get_status_message(cls, stats: RegistrationStatistics, verbose: bool = False time spent: {timedelta(seconds=stats.time_spent)}""" + \ (f""" time spent total: {stats.time_spent_total:.2f} s - time spent average: {timedelta(seconds=stats.time_average)} - """ if verbose else "") + f""" + time spent average: {timedelta(seconds=stats.time_average)}""" if verbose else "") + \ + f""" Difficulty: [bold white]{millify(stats.difficulty)}[/bold white] Iters: [bold white]{get_human_readable(int(stats.hash_rate), 'H')}/s[/bold white] Block: [bold white]{stats.block_number}[/bold white] From bf9794332222e1e312a358a04c31070b0741f22c Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 12 Oct 2022 11:47:12 -0400 Subject: [PATCH 23/27] allow --output_in_place --- bittensor/_subtensor/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bittensor/_subtensor/__init__.py b/bittensor/_subtensor/__init__.py index eab11fd1cc..11565fdbd6 100644 --- a/bittensor/_subtensor/__init__.py +++ b/bittensor/_subtensor/__init__.py @@ -192,7 +192,7 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ): # registration args. Used for register and re-register and anything that calls register. parser.add_argument('--' + prefix_str + 'subtensor.register.num_processes', '-n', dest=prefix_str + 'subtensor.register.num_processes', help="Number of processors to use for registration", type=int, default=bittensor.defaults.subtensor.register.num_processes) parser.add_argument('--' + prefix_str + 'subtensor.register.update_interval', '--' + prefix_str + 'subtensor.register.cuda.update_interval', '--' + prefix_str + 'cuda.update_interval', '-u', help="The number of nonces to process before checking for next block during registration", type=int, default=bittensor.defaults.subtensor.register.update_interval) - parser.add_argument('--' + prefix_str + 'subtensor.register.output_in_place', help="Whether to ouput the registration statistics in-place. Set flag to enable.", action='store_true', required=False, default=bittensor.defaults.subtensor.register.output_in_place) + parser.add_argument('--' + prefix_str + 'subtensor.register.output_in_place', '--' + prefix_str + 'output_in_place', help="Whether to ouput the registration statistics in-place. Set flag to enable.", action='store_true', required=False, default=bittensor.defaults.subtensor.register.output_in_place) parser.add_argument('--' + prefix_str + 'subtensor.register.verbose', help="Whether to ouput the registration statistics verbosely.", action='store_true', required=False, default=bittensor.defaults.subtensor.register.verbose) ## Registration args for CUDA registration. From da5d1b1c10d5b435b38aa088fcbb898c2aa1fbac Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 12 Oct 2022 11:49:59 -0400 Subject: [PATCH 24/27] fix n --- bittensor/utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 7ae9425f5e..9526ad41fc 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -796,7 +796,7 @@ def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: mu hash_rate_ = (num_time * TPB * update_interval) / time_since_last hash_rates.append(hash_rate_) hash_rates.pop(0) # remove the 0th data point - curr_stats.hash_rate = sum([hash_rates[i]*weights[i] for i in range(n)])/(sum(weights)) + curr_stats.hash_rate = sum([hash_rates[i]*weights[i] for i in range(n_samples)])/(sum(weights)) # update time last to now time_last = time_now From 304adbb2b42d6318ff6fb42b2a11d58053b77ef0 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 12 Oct 2022 11:51:13 -0400 Subject: [PATCH 25/27] change to --no_ouput_in_place --- bittensor/_subtensor/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bittensor/_subtensor/__init__.py b/bittensor/_subtensor/__init__.py index 11565fdbd6..cd60b673ac 100644 --- a/bittensor/_subtensor/__init__.py +++ b/bittensor/_subtensor/__init__.py @@ -192,7 +192,7 @@ def add_args(cls, parser: argparse.ArgumentParser, prefix: str = None ): # registration args. Used for register and re-register and anything that calls register. parser.add_argument('--' + prefix_str + 'subtensor.register.num_processes', '-n', dest=prefix_str + 'subtensor.register.num_processes', help="Number of processors to use for registration", type=int, default=bittensor.defaults.subtensor.register.num_processes) parser.add_argument('--' + prefix_str + 'subtensor.register.update_interval', '--' + prefix_str + 'subtensor.register.cuda.update_interval', '--' + prefix_str + 'cuda.update_interval', '-u', help="The number of nonces to process before checking for next block during registration", type=int, default=bittensor.defaults.subtensor.register.update_interval) - parser.add_argument('--' + prefix_str + 'subtensor.register.output_in_place', '--' + prefix_str + 'output_in_place', help="Whether to ouput the registration statistics in-place. Set flag to enable.", action='store_true', required=False, default=bittensor.defaults.subtensor.register.output_in_place) + parser.add_argument('--' + prefix_str + 'subtensor.register.no_output_in_place', '--' + prefix_str + 'no_output_in_place', dest="subtensor.register.output_in_place", help="Whether to not ouput the registration statistics in-place. Set flag to disable output in-place.", action='store_false', required=False, default=bittensor.defaults.subtensor.register.output_in_place) parser.add_argument('--' + prefix_str + 'subtensor.register.verbose', help="Whether to ouput the registration statistics verbosely.", action='store_true', required=False, default=bittensor.defaults.subtensor.register.verbose) ## Registration args for CUDA registration. From 147e853b99f937fe3bceeee93c9dc1219b303928 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Wed, 12 Oct 2022 11:56:53 -0400 Subject: [PATCH 26/27] fix test --- tests/unit_tests/bittensor_tests/utils/test_utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/unit_tests/bittensor_tests/utils/test_utils.py b/tests/unit_tests/bittensor_tests/utils/test_utils.py index 669877d62e..030cdbfb83 100644 --- a/tests/unit_tests/bittensor_tests/utils/test_utils.py +++ b/tests/unit_tests/bittensor_tests/utils/test_utils.py @@ -415,13 +415,14 @@ class MockException(Exception): update_interval=update_interval, stopEvent=MagicMock(is_set=MagicMock(return_value=False)), newBlockEvent=MagicMock(is_set=MagicMock(return_value=False)), - time_queue=MagicMock(put_nowait=MagicMock()), - limit=10000 + finished_queue=MagicMock(put=MagicMock()), + limit=10000, + proc_num=0, ) with patch('bittensor.utils.solve_for_nonce_block_cuda', - side_effect=[(None, 10.0), MockException] # first call returns mocked no solution, second call raises exception + side_effect=[None, MockException] # first call returns mocked no solution, second call raises exception ) as mock_solve_for_nonce_block_cuda: # Should exit early From e059531ab0d4e250a4949a7e0a11ac4b736ccfe9 Mon Sep 17 00:00:00 2001 From: Cameron Fairchild Date: Thu, 13 Oct 2022 10:18:31 -0400 Subject: [PATCH 27/27] fix ref to solution --- bittensor/utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 9526ad41fc..8482d5a336 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -515,7 +515,6 @@ def solve_for_difficulty_fast( subtensor, wallet, output_in_place: bool = True, logger.start() solution = None - hash_rate = 0 # EWMA hash_rate (H/s) hash_rates = [0] * n_samples # The last n true hash_rates weights = [alpha_ ** i for i in range(n_samples)] # weights decay by alpha @@ -746,6 +745,7 @@ def update_curr_block(block_number: int, block_bytes: bytes, diff: int, lock: mu hash_rates = [0] * n_samples # The last n true hash_rates weights = [alpha_ ** i for i in range(n_samples)] # weights decay by alpha + solution = None while not wallet.is_registered(subtensor): # Wait until a solver finds a solution try: