From 4ba7ab1cb35e1595d62dbab30b97aea3d9e2e75b Mon Sep 17 00:00:00 2001 From: Thebes Date: Mon, 7 Nov 2022 10:09:12 -0500 Subject: [PATCH 1/2] initial commit --- bittensor/_axon/axon_impl.py | 44 ++++++++++++++++------------ bittensor/_dendrite/dendrite_impl.py | 25 +++++++++++----- 2 files changed, 43 insertions(+), 26 deletions(-) diff --git a/bittensor/_axon/axon_impl.py b/bittensor/_axon/axon_impl.py index ca63a2cb81..cadcbcaeca 100644 --- a/bittensor/_axon/axon_impl.py +++ b/bittensor/_axon/axon_impl.py @@ -105,25 +105,33 @@ def __init__( self.priority = priority self.priority_threadpool= priority_threadpool - # == Prometheus - # We are running over various suffix values in the event that there are multiple axons in the same process. - # The first axon is created with a null suffix and subsequent values are ordered like so: axon_is_started, axon_is_started_1, axon_is_started_2 etc... - + # == Prometheus if self.prometheus_level != bittensor.prometheus.level.OFF.name: - registry = CollectorRegistry() - self.is_started = Enum('axon_is_started', 'is_started', states=['stopped', 'started'], registry=registry) - self.total_forward = Counter('axon_total_forward', 'total_forward', registry=registry) - self.total_backward = Counter('axon_total_backward', 'total_backward', registry=registry) - self.forward_latency = Histogram('axon_forward_latency', 'forward_latency', buckets=list(range(0,bittensor.__blocktime__,1)), registry=registry) - self.backward_latency = Histogram('axon_backward_latency', 'backward_latency', buckets=list(range(0,bittensor.__blocktime__,1)), registry=registry) - self.forward_synapses = Counter('axon_forward_synapses', 'forward_synapses', ["synapse"], registry=registry) - self.backward_synapses = Counter('axon_backward_synapses', 'backward_synapses', ["synapse"], registry=registry) - self.forward_codes = Counter('axon_forward_codes', 'forward_codes', ["code"], registry=registry) - self.backward_codes = Counter('axon_backward_codes', 'backward_codes', ["code"], registry=registry) - self.forward_hotkeys = Counter('axon_forward_hotkeys', 'forward_hotkeys', ["hotkey"], registry=registry) - self.backward_hotkeys = Counter('axon_backward_hotkeys', 'backward_hotkeys', ["hotkey"], registry=registry) - self.forward_bytes = Counter('axon_forward_bytes', 'forward_bytes', ["hotkey"], registry=registry) - self.backward_bytes = Counter('axon_backward_bytes', 'backward_bytes', ["hotkey"], registry=registry) + # We are running over various suffix values in the event that there are multiple axons in the same process. + # The first axon is created with a null suffix and subsequent values are ordered like so: axon_is_started, axon_is_started_1, axon_is_started_2 etc... + # This avoids overlap such that further axons created on the same process are numbered with suffixes in order. + suffix = "" + suffix_increment = 0 + while True: + try: + self.is_started = Enum('axon_is_started{}'.format(suffix), 'is_started', states=['stopped', 'started']) + self.total_forward = Counter('axon_total_forward{}'.format(suffix), 'total_forward') + self.total_backward = Counter('axon_total_backward{}'.format(suffix), 'total_backward') + self.forward_latency = Histogram('axon_forward_latency{}'.format(suffix), 'forward_latency', buckets=list(range(0,bittensor.__blocktime__,1))) + self.backward_latency = Histogram('axon_backward_latency{}'.format(suffix), 'backward_latency', buckets=list(range(0,bittensor.__blocktime__,1))) + self.forward_synapses = Counter('axon_forward_synapses{}'.format(suffix), 'forward_synapses', ["synapse"]) + self.backward_synapses = Counter('axon_backward_synapses{}'.format(suffix), 'backward_synapses', ["synapse"]) + self.forward_codes = Counter('axon_forward_codes{}'.format(suffix), 'forward_codes', ["code"]) + self.backward_codes = Counter('axon_backward_codes{}'.format(suffix), 'backward_codes', ["code"]) + self.forward_hotkeys = Counter('axon_forward_hotkeys{}'.format(suffix), 'forward_hotkeys', ["hotkey"]) + self.backward_hotkeys = Counter('axon_backward_hotkeys{}'.format(suffix), 'backward_hotkeys', ["hotkey"]) + self.forward_bytes = Counter('axon_forward_bytes{}'.format(suffix), 'forward_bytes', ["hotkey"]) + self.backward_bytes = Counter('axon_backward_bytes{}'.format(suffix), 'backward_bytes', ["hotkey"]) + except ValueError: + suffix_increment += 1 + suffix = "_{}".format(suffix_increment) + continue + break def __str__(self) -> str: return "Axon({}, {}, {}, {})".format( self.ip, self.port, self.wallet.hotkey.ss58_address, "started" if self.started else "stopped") diff --git a/bittensor/_dendrite/dendrite_impl.py b/bittensor/_dendrite/dendrite_impl.py index 75253b6790..dcbb47e48b 100644 --- a/bittensor/_dendrite/dendrite_impl.py +++ b/bittensor/_dendrite/dendrite_impl.py @@ -86,15 +86,24 @@ def __init__( self.stats = self._init_stats() # == Prometheus - # We are running over various suffix values in the event that there are multiple dendrites in the same process. - # The first dendrite is created with a null suffix. Values are ordered like so: dendrite_counters, dendrite_counters_1, dendrite_counters_2 etc... if self.config.dendrite.prometheus.level != bittensor.prometheus.level.OFF.name: - registry = CollectorRegistry() - self.prometheus_counters = Counter('dendrite_counters', 'dendrite_counters', ['name'], registry=registry) - self.prometheus_latency = Histogram('dendrite_latency', 'dendrite_latency', buckets=list(range(0,bittensor.__blocktime__,1)), registry=registry) - self.prometheus_latency_per_uid = Summary('dendrite_latency_per_uid', 'dendrite_latency_per_uid', ['uid'], registry=registry) - self.prometheus_successes_per_uid = Counter('dendrite_successes_per_uid', 'dendrite_successes_per_uid', ['uid'], registry=registry) - self.prometheus_failures_per_uid = Counter('dendrite_failures_per_uid', 'dendrite_failures_per_uid', ['uid'], registry=registry) + # We are running over various suffix values in the event that there are multiple dendrites in the same process. + # The first dendrite is created with a null suffix. Values are ordered like so: dendrite_counters, dendrite_counters_1, dendrite_counters_2 etc... + # This avoids overlap such that further dendrites created on the same process are numbered with suffixes in order. + suffix = "" + suffix_increment = 0 + while True: + try: + self.prometheus_counters = Counter('dendrite_counters{}'.format(suffix), 'dendrite_counters', ['name']) + self.prometheus_latency = Histogram('dendrite_latency{}'.format(suffix), 'dendrite_latency', buckets=list(range(0,bittensor.__blocktime__,1))) + self.prometheus_latency_per_uid = Summary('dendrite_latency_per_uid{}'.format(suffix), 'dendrite_latency_per_uid', ['uid']) + self.prometheus_successes_per_uid = Counter('dendrite_successes_per_uid{}'.format(suffix), 'dendrite_successes_per_uid', ['uid']) + self.prometheus_failures_per_uid = Counter('dendrite_failures_per_uid{}'.format(suffix), 'dendrite_failures_per_uid', ['uid']) + except ValueError: + suffix_increment += 1 + suffix = "_{}".format(suffix_increment) + continue + break def __str__(self): return "Dendrite({}, {})".format(self.wallet.hotkey.ss58_address, self.receptor_pool) From ee287668bee3abedba46cfa1e9b33e51b4c44f2b Mon Sep 17 00:00:00 2001 From: Thebes Date: Mon, 7 Nov 2022 14:45:14 -0500 Subject: [PATCH 2/2] promo change to axon and dendrite --- bittensor/_axon/axon_impl.py | 76 ++++++++++-------------- bittensor/_dendrite/dendrite_impl.py | 60 ++++++++----------- tests/integration_tests/test_dendrite.py | 2 +- 3 files changed, 57 insertions(+), 81 deletions(-) diff --git a/bittensor/_axon/axon_impl.py b/bittensor/_axon/axon_impl.py index cadcbcaeca..f059ac891a 100644 --- a/bittensor/_axon/axon_impl.py +++ b/bittensor/_axon/axon_impl.py @@ -27,11 +27,11 @@ import grpc import wandb import pandas +import uuid from loguru import logger import torch.nn.functional as F import concurrent -from prometheus_client import Counter, Histogram, Enum, CollectorRegistry import bittensor import bittensor.utils.stats as stat_utils @@ -39,6 +39,21 @@ logger = logger.opt(colors=True) +from prometheus_client import Counter, Histogram, Enum, CollectorRegistry +PROM_axon_is_started = Enum('axon_is_started', 'is_started', states=['stopped', 'started']) +PROM_total_forward = Counter('axon_total_forward', 'total_forward', ['wallet', 'identifier']) +PROM_total_backward = Counter('axon_total_backward', 'total_backward', ['wallet', 'identifier']) +PROM_forward_latency = Histogram('axon_forward_latency', 'forward_latency', ['wallet', 'identifier'], buckets=list(range(0,bittensor.__blocktime__,1))) +PROM_backward_latency = Histogram('axon_backward_latency', 'backward_latency', ['wallet', 'identifier'], buckets=list(range(0,bittensor.__blocktime__,1))) +PROM_forward_synapses = Counter('axon_forward_synapses', 'forward_synapses', ['wallet', 'identifier', "synapse"]) +PROM_backward_synapses = Counter('axon_backward_synapses', 'backward_synapses', ['wallet', 'identifier', "synapse"]) +PROM_forward_codes = Counter('axon_forward_codes', 'forward_codes', ['wallet', 'identifier', "code"]) +PROM_backward_codes = Counter('axon_backward_codes', 'backward_codes', ['wallet', 'identifier', "code"]) +PROM_forward_hotkeys = Counter('axon_forward_hotkeys', 'forward_hotkeys', ['wallet', 'identifier', "hotkey"]) +PROM_backward_hotkeys = Counter('axon_backward_hotkeys', 'backward_hotkeys', ['wallet', 'identifier', "hotkey"]) +PROM_forward_bytes = Counter('axon_forward_bytes', 'forward_bytes', ['wallet', 'identifier', "hotkey"]) +PROM_backward_bytes = Counter('axon_backward_bytes', 'backward_bytes', ['wallet', 'identifier', "hotkey"]) + class Axon( bittensor.grpc.BittensorServicer ): r""" Services Forward and Backward requests from other neurons. """ @@ -103,35 +118,8 @@ def __init__( # -- Priority self.priority = priority - self.priority_threadpool= priority_threadpool - - # == Prometheus - if self.prometheus_level != bittensor.prometheus.level.OFF.name: - # We are running over various suffix values in the event that there are multiple axons in the same process. - # The first axon is created with a null suffix and subsequent values are ordered like so: axon_is_started, axon_is_started_1, axon_is_started_2 etc... - # This avoids overlap such that further axons created on the same process are numbered with suffixes in order. - suffix = "" - suffix_increment = 0 - while True: - try: - self.is_started = Enum('axon_is_started{}'.format(suffix), 'is_started', states=['stopped', 'started']) - self.total_forward = Counter('axon_total_forward{}'.format(suffix), 'total_forward') - self.total_backward = Counter('axon_total_backward{}'.format(suffix), 'total_backward') - self.forward_latency = Histogram('axon_forward_latency{}'.format(suffix), 'forward_latency', buckets=list(range(0,bittensor.__blocktime__,1))) - self.backward_latency = Histogram('axon_backward_latency{}'.format(suffix), 'backward_latency', buckets=list(range(0,bittensor.__blocktime__,1))) - self.forward_synapses = Counter('axon_forward_synapses{}'.format(suffix), 'forward_synapses', ["synapse"]) - self.backward_synapses = Counter('axon_backward_synapses{}'.format(suffix), 'backward_synapses', ["synapse"]) - self.forward_codes = Counter('axon_forward_codes{}'.format(suffix), 'forward_codes', ["code"]) - self.backward_codes = Counter('axon_backward_codes{}'.format(suffix), 'backward_codes', ["code"]) - self.forward_hotkeys = Counter('axon_forward_hotkeys{}'.format(suffix), 'forward_hotkeys', ["hotkey"]) - self.backward_hotkeys = Counter('axon_backward_hotkeys{}'.format(suffix), 'backward_hotkeys', ["hotkey"]) - self.forward_bytes = Counter('axon_forward_bytes{}'.format(suffix), 'forward_bytes', ["hotkey"]) - self.backward_bytes = Counter('axon_backward_bytes{}'.format(suffix), 'backward_bytes', ["hotkey"]) - except ValueError: - suffix_increment += 1 - suffix = "_{}".format(suffix_increment) - continue - break + self.priority_threadpool = priority_threadpool + self._prometheus_uuid = uuid.uuid1() def __str__(self) -> str: return "Axon({}, {}, {}, {})".format( self.ip, self.port, self.wallet.hotkey.ss58_address, "started" if self.started else "stopped") @@ -247,17 +235,17 @@ def check_if_should_return() -> bool: def finalize_codes_stats_and_logs( message = None): # === Prometheus if self.prometheus_level != bittensor.prometheus.level.OFF.name: - self.total_forward.inc() - self.forward_latency.observe( clock.time() - start_time ) + PROM_total_forward.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid ).inc() + PROM_forward_latency.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid ).observe( clock.time() - start_time ) if self.prometheus_level == bittensor.prometheus.level.DEBUG.name: - self.forward_hotkeys.labels( request.hotkey ).inc() - self.forward_bytes.labels( request.hotkey ).inc( sys.getsizeof( request ) ) + PROM_forward_hotkeys.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid, hotkey = request.hotkey ).inc() + PROM_forward_bytes.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid, hotkey = request.hotkey ).inc( sys.getsizeof( request ) ) for index, synapse in enumerate( synapses ): # === Prometheus if self.prometheus_level != bittensor.prometheus.level.OFF.name: - self.forward_synapses.labels( str(synapse) ).inc() - self.forward_codes.labels( str(synapse_codes[ index ]) ).inc() + PROM_forward_synapses.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid, synapse = str(synapse) ).inc() + PROM_forward_codes.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid, code = str(synapse_codes[ index ]) ).inc() # === Logging request.synapses [ index ].return_code = synapse_codes[ index ] # Set synapse wire proto codes. @@ -479,17 +467,17 @@ def check_if_should_return() -> bool: def finalize_codes_stats_and_logs(): # === Prometheus if self.prometheus_level != bittensor.prometheus.level.OFF.name: - self.total_backward.inc() - self.backward_latency.observe( clock.time() - start_time ) + PROM_total_backward.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid ).inc() + PROM_backward_latency.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid ).observe( clock.time() - start_time ) if self.prometheus_level == bittensor.prometheus.level.DEBUG.name: - self.backward_hotkeys.labels( request.hotkey ).inc() - self.backward_bytes.labels( request.hotkey ).inc( sys.getsizeof( request ) ) + PROM_backward_hotkeys.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid, hotkey = request.hotkey ).inc() + PROM_backward_bytes.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid, hotkey = request.hotkey ).inc( sys.getsizeof( request ) ) for index, synapse in enumerate( synapses ): # === Prometheus if self.prometheus_level != bittensor.prometheus.level.OFF.name: - self.backward_synapses.labels( str(synapse) ).inc() - self.backward_codes.labels( str(synapse_codes[ index ]) ).inc() + PROM_backward_synapses.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid, synapse = str(synapse) ).inc() + PROM_backward_codes.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid, code = str(synapse_codes[ index ]) ).inc() # === Logging request.synapses [ index ].return_code = synapse_codes[ index ] # Set synapse wire proto codes. @@ -826,7 +814,7 @@ def start(self) -> 'Axon': # Switch prometheus ENUM. if self.prometheus_level != bittensor.prometheus.level.OFF.name: - self.is_started.state('started') + PROM_axon_is_started.state('started') return self @@ -840,7 +828,7 @@ def stop(self) -> 'Axon': # Switch prometheus ENUM. if self.prometheus_level != bittensor.prometheus.level.OFF.name: - self.is_started.state('stopped') + PROM_axon_is_started.state('stopped') return self diff --git a/bittensor/_dendrite/dendrite_impl.py b/bittensor/_dendrite/dendrite_impl.py index dcbb47e48b..3501da9212 100644 --- a/bittensor/_dendrite/dendrite_impl.py +++ b/bittensor/_dendrite/dendrite_impl.py @@ -25,6 +25,7 @@ import pandas import random import time +import uuid from torch.autograd.function import once_differentiable from loguru import logger @@ -40,13 +41,19 @@ import wandb -from prometheus_client import Summary, Counter, Histogram, CollectorRegistry logger = logger.opt(colors=True) # dummy tensor that triggers autograd DUMMY = torch.empty(0, requires_grad=True) +# Global prometheus +from prometheus_client import Summary, Counter, Histogram, CollectorRegistry +PROM_prometheus_counters = Counter('dendrite_counters', 'dendrite_counters', ['wallet', 'identifier', 'name']) +PROM_prometheus_latency = Histogram('dendrite_latency', 'dendrite_latency', ['wallet', 'identifier'], buckets=list(range(0,bittensor.__blocktime__,1))) +PROM_prometheus_latency_per_uid = Summary('dendrite_latency_per_uid', 'dendrite_latency_per_uid', ['wallet', 'identifier', 'uid']) +PROM_prometheus_successes_per_uid = Counter('dendrite_successes_per_uid', 'dendrite_successes_per_uid', ['wallet', 'identifier', 'uid']) +PROM_prometheus_failures_per_uid = Counter('dendrite_failures_per_uid', 'dendrite_failures_per_uid', ['wallet', 'identifier', 'uid']) class Dendrite(torch.autograd.Function): r""" This is the implementation class for a bittensor.dendrite(). The dendrite class operates as a normal torch autograd friendly operation @@ -57,7 +64,7 @@ class Dendrite(torch.autograd.Function): Args: config (:obj:`bittensor.Config`, `optional`, defaults to bittensor.dendrite.config()): config namespace object created by calling bittensor.dendrite.config() - wallet (:obj:`bittensor.Wallet`, `optional`, defaults to bittensor.wallet( name = 'default', hotkey = 'default')): + wallet (:obj:`bittensor.Wallet`, `optional`, defaults to bittensor.wallet( name = 'default', wallet ='default')): A bittensor wallet object containing a pair of cryptographic keys, the hot and coldkey, used for signing messages on the wire. receptor_pool (:obj:`bittensor.ReceptorPool`, `optional`, defaults to bittensor.receptor_pool()): @@ -84,26 +91,7 @@ def __init__( # ---- Dendrite stats # num of time we have sent request to a peer, received successful respond, and the respond time self.stats = self._init_stats() - - # == Prometheus - if self.config.dendrite.prometheus.level != bittensor.prometheus.level.OFF.name: - # We are running over various suffix values in the event that there are multiple dendrites in the same process. - # The first dendrite is created with a null suffix. Values are ordered like so: dendrite_counters, dendrite_counters_1, dendrite_counters_2 etc... - # This avoids overlap such that further dendrites created on the same process are numbered with suffixes in order. - suffix = "" - suffix_increment = 0 - while True: - try: - self.prometheus_counters = Counter('dendrite_counters{}'.format(suffix), 'dendrite_counters', ['name']) - self.prometheus_latency = Histogram('dendrite_latency{}'.format(suffix), 'dendrite_latency', buckets=list(range(0,bittensor.__blocktime__,1))) - self.prometheus_latency_per_uid = Summary('dendrite_latency_per_uid{}'.format(suffix), 'dendrite_latency_per_uid', ['uid']) - self.prometheus_successes_per_uid = Counter('dendrite_successes_per_uid{}'.format(suffix), 'dendrite_successes_per_uid', ['uid']) - self.prometheus_failures_per_uid = Counter('dendrite_failures_per_uid{}'.format(suffix), 'dendrite_failures_per_uid', ['uid']) - except ValueError: - suffix_increment += 1 - suffix = "_{}".format(suffix_increment) - continue - break + self._prometheus_uuid = uuid.uuid1() def __str__(self): return "Dendrite({}, {})".format(self.wallet.hotkey.ss58_address, self.receptor_pool) @@ -322,16 +310,16 @@ def _forward( outputs: List[torch.Tensor] = forward_response[2:] packed_outputs: List[ List[torch.Tensor] ] = [ outputs[ s : s + len(synapses) ] for s in range (0, len(outputs), len( synapses )) ] - # === Prometheus counters. + # === Prometheus counters. if self.config.dendrite.prometheus.level != bittensor.prometheus.level.OFF.name: - self.prometheus_counters.labels( 'total_requests' ).inc() - self.prometheus_counters.labels( 'total_endpoint_requests' ).inc( len(endpoints) ) - self.prometheus_counters.labels( 'total_request_bytes' ).inc( sum(p.element_size() * p.nelement() for p in inputs) ) - self.prometheus_counters.labels( 'total_request_params' ).inc( sum(p.numel() for p in inputs) ) + PROM_prometheus_counters.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid, name = 'total_requests' ).inc() + PROM_prometheus_counters.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid, name = 'total_endpoint_requests' ).inc( len(endpoints) ) + PROM_prometheus_counters.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid, name = 'total_request_bytes' ).inc( sum(p.element_size() * p.nelement() for p in inputs) ) + PROM_prometheus_counters.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid, name = 'total_request_params' ).inc( sum(p.numel() for p in inputs) ) # Capture synapses. for synapse in enumerate( synapses ): - self.prometheus_counters.labels( str(synapse) ).inc() + PROM_prometheus_counters.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid, name = str(synapse) ).inc() for i in range(len(endpoints)): n_success = (codes[i] == 1).sum().item() @@ -339,23 +327,23 @@ def _forward( response_time = times[i].mean().item() # Capture outputs. - self.prometheus_counters.labels( 'total_response_bytes' ).inc( sum(p.element_size() * p.nelement() for p in outputs[i]) ) - self.prometheus_counters.labels( 'total_response_params' ).inc( sum(p.numel() for p in outputs[i]) ) + PROM_prometheus_counters.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid, name = 'total_response_bytes' ).inc( sum(p.element_size() * p.nelement() for p in outputs[i]) ) + PROM_prometheus_counters.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid, name = 'total_response_params' ).inc( sum(p.numel() for p in outputs[i]) ) # Capture global success rates. if is_success: - self.prometheus_counters.labels( 'total_success' ).inc() - self.prometheus_latency.observe( response_time ) + PROM_prometheus_counters.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid, name = 'total_success' ).inc() + PROM_prometheus_latency.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid).observe( response_time ) else: - self.prometheus_counters.labels( 'total_failure' ).inc() + PROM_prometheus_counters.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid, name = 'total_failure' ).inc() # === Prometheus DEBUG (per uid info.) if self.config.dendrite.prometheus.level == bittensor.prometheus.level.DEBUG.name: if is_success: - self.prometheus_latency_per_uid.labels(str(endpoints[i].uid)).observe( response_time ) - self.prometheus_successes_per_uid.labels(str(endpoints[i].uid)).inc() + PROM_prometheus_latency_per_uid.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid, uid = str(endpoints[i].uid) ).observe( response_time ) + PROM_prometheus_successes_per_uid.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid, uid = str(endpoints[i].uid) ).inc() else: - self.prometheus_failures_per_uid.labels(str(endpoints[i].uid)).inc() + PROM_prometheus_failures_per_uid.labels( wallet = self.wallet.hotkey.ss58_address, identifier = self._prometheus_uuid, uid = str(endpoints[i].uid) ).inc() return packed_outputs, packed_codes, packed_times diff --git a/tests/integration_tests/test_dendrite.py b/tests/integration_tests/test_dendrite.py index ed253def54..c4265065d8 100644 --- a/tests/integration_tests/test_dendrite.py +++ b/tests/integration_tests/test_dendrite.py @@ -285,7 +285,7 @@ def forward_casual_lm_next(inputs_x, synapse, model_output=None): axon.attach_synapse_callback( forward_hidden_state, synapse_type = bittensor.proto.Synapse.SynapseType.TEXT_LAST_HIDDEN_STATE ) axon.attach_synapse_callback( forward_generate, synapse_type = bittensor.proto.Synapse.SynapseType.TEXT_SEQ_2_SEQ ) axon.attach_synapse_callback( forward_casual_lm, synapse_type = bittensor.proto.Synapse.SynapseType.TEXT_CAUSAL_LM ) - axon.attach_synapse_callback(forward_casual_lm_next, synapse_type=bittensor.proto.Synapse.SynapseType.TEXT_CAUSAL_LM_NEXT) + axon.attach_synapse_callback( forward_casual_lm_next, synapse_type=bittensor.proto.Synapse.SynapseType.TEXT_CAUSAL_LM_NEXT) axon.start() endpoint = bittensor.endpoint(