Split expired message metrics into sync and async (ava-labs#1550)

snow/networking/handler/handler.go

@@ -8,6 +8,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
+
 	"github.com/ava-labs/avalanchego/api/health"
 	"github.com/ava-labs/avalanchego/ids"
 	"github.com/ava-labs/avalanchego/message"
@@ -284,7 +286,7 @@ func (h *handler) dispatchSync() {
 	for {
 		// Get the next message we should process. If the handler is shutting
 		// down, we may fail to pop a message.
-		msg, ok := h.popUnexpiredMsg(h.syncMessageQueue)
+		msg, ok := h.popUnexpiredMsg(h.syncMessageQueue, h.metrics.expired)
 		if !ok {
 			return
 		}
@@ -311,7 +313,7 @@ func (h *handler) dispatchAsync() {
 	for {
 		// Get the next message we should process. If the handler is shutting
 		// down, we may fail to pop a message.
-		msg, ok := h.popUnexpiredMsg(h.asyncMessageQueue)
+		msg, ok := h.popUnexpiredMsg(h.asyncMessageQueue, h.metrics.asyncExpired)
 		if !ok {
 			return
 		}
@@ -673,7 +675,7 @@ func (h *handler) getEngine() (common.Engine, error) {
 	}
 }
 
-func (h *handler) popUnexpiredMsg(queue MessageQueue) (message.InboundMessage, bool) {
+func (h *handler) popUnexpiredMsg(queue MessageQueue, expired prometheus.Counter) (message.InboundMessage, bool) {
 	for {
 		// Get the next message we should process. If the handler is shutting
 		// down, we may fail to pop a message.
@@ -688,7 +690,7 @@ func (h *handler) popUnexpiredMsg(queue MessageQueue) (message.InboundMessage, b
 				"Dropping message from %s due to timeout: %s",
 				msg.NodeID(), msg,
 			)
-			h.metrics.expired.Inc()
+			expired.Inc()
 			msg.OnFinishedHandling()
 			continue
 		}

snow/networking/handler/metrics.go

@@ -14,8 +14,9 @@ import (
 )
 
 type metrics struct {
-	expired  prometheus.Counter
-	messages map[message.Op]metric.Averager
+	expired      prometheus.Counter
+	asyncExpired prometheus.Counter
+	messages     map[message.Op]metric.Averager
 }
 
 func newMetrics(namespace string, reg prometheus.Registerer) (*metrics, error) {
@@ -24,9 +25,17 @@ func newMetrics(namespace string, reg prometheus.Registerer) (*metrics, error) {
 	expired := prometheus.NewCounter(prometheus.CounterOpts{
 		Namespace: namespace,
 		Name:      "expired",
-		Help:      "Incoming messages dropped because the message deadline expired",
+		Help:      "Incoming sync messages dropped because the message deadline expired",
 	})
-	errs.Add(reg.Register(expired))
+	asyncExpired := prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: namespace,
+		Name:      "async_expired",
+		Help:      "Incoming async messages dropped because the message deadline expired",
+	})
+	errs.Add(
+		reg.Register(expired),
+		reg.Register(asyncExpired),
+	)
 
 	messages := make(map[message.Op]metric.Averager, len(message.ConsensusOps))
 	for _, op := range message.ConsensusOps {
@@ -41,7 +50,8 @@ func newMetrics(namespace string, reg prometheus.Registerer) (*metrics, error) {
 	}
 
 	return &metrics{
-		expired:  expired,
-		messages: messages,
+		expired:      expired,
+		asyncExpired: asyncExpired,
+		messages:     messages,
 	}, errs.Err
 }