metrics: Add metric for query blocks behind

docs/metrics.md

@@ -57,6 +57,9 @@ the **maximum size of a query result** (in CacheWeight)
 the **size of the result of successful GraphQL queries** (in CacheWeight)
 - `query_semaphore_wait_ms`
 Moving **average of time spent on waiting for postgres query semaphore**
+- `query_blocks_behind`
+A histogram for how many blocks behind the subgraph head queries are being made at.
+This helps inform pruning decisions.
 - `query_kill_rate`
 The rate at which the load manager kills queries
 - `registered_metrics`
@@ -66,4 +69,4 @@ The **number of Postgres connections** currently **checked out**
 - `store_connection_error_count`
 The **number of Postgres connections errors**
 - `store_connection_wait_time_ms`
-**Average connection wait time**
+**Average connection wait time**

graph/src/components/graphql.rs

@@ -49,4 +49,5 @@ pub trait GraphQLMetrics: Send + Sync + 'static {
     fn observe_query_parsing(&self, duration: Duration, results: &QueryResults);
     fn observe_query_validation(&self, duration: Duration, id: &DeploymentHash);
     fn observe_query_validation_error(&self, error_codes: Vec<&str>, id: &DeploymentHash);
+    fn observe_query_blocks_behind(&self, blocks_behind: i32, id: &DeploymentHash);
 }

graphql/src/metrics.rs

@@ -14,6 +14,7 @@ pub struct GraphQLMetrics {
     query_result_size: Box<Histogram>,
     query_result_size_max: Box<Gauge>,
     query_validation_error_counter: Box<CounterVec>,
+    query_blocks_behind: Box<HistogramVec>,
 }
 
 impl fmt::Debug for GraphQLMetrics {
@@ -73,6 +74,12 @@ impl GraphQLMetricsTrait for GraphQLMetrics {
                 .inc();
         }
     }
+
+    fn observe_query_blocks_behind(&self, blocks_behind: i32, id: &DeploymentHash) {
+        self.query_blocks_behind
+            .with_label_values(&[id.as_str()])
+            .observe(blocks_behind as f64);
+    }
 }
 
 impl GraphQLMetrics {
@@ -128,13 +135,26 @@ impl GraphQLMetrics {
             )
             .unwrap();
 
+        let query_blocks_behind = registry
+            .new_histogram_vec(
+                "query_blocks_behind",
+                "How many blocks the query block is behind the subgraph head",
+                vec![String::from("deployment")],
+                vec![
+                    0.0, 5.0, 10.0, 20.0, 30.0, 40.0, 50.0, 100.0, 200.0, 500.0, 1000.0, 10000.0,
+                    100000.0, 1000000.0, 10000000.0,
+                ],
+            )
+            .unwrap();
+
         Self {
             query_execution_time,
             query_parsing_time,
             query_validation_time,
             query_result_size,
             query_result_size_max,
             query_validation_error_counter,
+            query_blocks_behind,
         }
     }
 

graphql/src/store/resolver.rs

@@ -2,6 +2,7 @@ use std::collections::BTreeMap;
 use std::result;
 use std::sync::Arc;
 
+use graph::components::graphql::GraphQLMetrics as _;
 use graph::components::store::{QueryPermit, SubscriptionManager, UnitStream};
 use graph::data::graphql::load_manager::LoadManager;
 use graph::data::graphql::{object, ObjectOrInterface};
@@ -105,6 +106,9 @@ impl StoreResolver {
         let store_clone = store.cheap_clone();
         let block_ptr = Self::locate_block(store_clone.as_ref(), bc, state).await?;
 
+        let blocks_behind = state.latest_block.number - block_ptr.ptr.number;
+        graphql_metrics.observe_query_blocks_behind(blocks_behind, &deployment);
+
         let has_non_fatal_errors = store
             .has_deterministic_errors(block_ptr.ptr.block_number())
             .await?;

server/http/src/service.rs

@@ -488,6 +488,7 @@ mod tests {
         fn observe_query_parsing(&self, _duration: Duration, _results: &QueryResults) {}
         fn observe_query_validation(&self, _duration: Duration, _id: &DeploymentHash) {}
         fn observe_query_validation_error(&self, _error_codes: Vec<&str>, _id: &DeploymentHash) {}
+        fn observe_query_blocks_behind(&self, _blocks_behind: i32, _id: &DeploymentHash) {}
     }
 
     #[async_trait]

server/http/tests/server.rs

@@ -20,6 +20,7 @@ impl GraphQLMetrics for TestGraphQLMetrics {
     fn observe_query_parsing(&self, _duration: Duration, _results: &QueryResults) {}
     fn observe_query_validation(&self, _duration: Duration, _id: &DeploymentHash) {}
     fn observe_query_validation_error(&self, _error_codes: Vec<&str>, _id: &DeploymentHash) {}
+    fn observe_query_blocks_behind(&self, _blocks_behind: i32, _id: &DeploymentHash) {}
 }
 
 /// A simple stupid query runner for testing.

server/index-node/src/service.rs

@@ -28,6 +28,7 @@ impl GraphQLMetrics for NoopGraphQLMetrics {
     fn observe_query_parsing(&self, _duration: Duration, _results: &QueryResults) {}
     fn observe_query_validation(&self, _duration: Duration, _id: &DeploymentHash) {}
     fn observe_query_validation_error(&self, _error_codes: Vec<&str>, _id: &DeploymentHash) {}
+    fn observe_query_blocks_behind(&self, _blocks_behind: i32, _id: &DeploymentHash) {}
 }
 
 /// An asynchronous response to a GraphQL request.