From 1525346c487c1058d823617ec3443e9bc7f65c86 Mon Sep 17 00:00:00 2001 From: Sergei Shulepov Date: Fri, 12 Nov 2021 12:29:49 +0000 Subject: [PATCH 1/2] Increase preparation-timeout to 60 seconds --- node/core/pvf/src/prepare/worker.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/core/pvf/src/prepare/worker.rs b/node/core/pvf/src/prepare/worker.rs index a8bb3516e296..6ce8a8b91764 100644 --- a/node/core/pvf/src/prepare/worker.rs +++ b/node/core/pvf/src/prepare/worker.rs @@ -37,7 +37,7 @@ use std::{sync::Arc, time::Duration}; const NICENESS_BACKGROUND: i32 = 10; const NICENESS_FOREGROUND: i32 = 0; -const COMPILATION_TIMEOUT: Duration = Duration::from_secs(10); +const COMPILATION_TIMEOUT: Duration = Duration::from_secs(60); /// Spawns a new worker with the given program path that acts as the worker and the spawn timeout. /// From a5ec35486b4800a67b9c85dc08d742dca78a1e2d Mon Sep 17 00:00:00 2001 From: Sergei Shulepov Date: Fri, 12 Nov 2021 14:19:08 +0000 Subject: [PATCH 2/2] Adapt `pvf_preparation_time` metric to the new value --- node/core/pvf/src/metrics.rs | 15 ++++++++++++++- node/core/pvf/src/prepare/worker.rs | 2 ++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/node/core/pvf/src/metrics.rs b/node/core/pvf/src/metrics.rs index 031397158efc..7a65b2f19466 100644 --- a/node/core/pvf/src/metrics.rs +++ b/node/core/pvf/src/metrics.rs @@ -152,8 +152,21 @@ impl metrics::Metrics for Metrics { prometheus::Histogram::with_opts( prometheus::HistogramOpts::new( "pvf_preparation_time", - "Time spent in preparing PVF artifacts", + "Time spent in preparing PVF artifacts in seconds", ) + .buckets(vec![ + // This is synchronized with COMPILATION_TIMEOUT=60s constant found in + // src/prepare/worker.rs + 0.1, + 0.5, + 1.0, + 10.0, + 20.0, + 30.0, + 40.0, + 50.0, + 60.0, + ]), )?, registry, )?, diff --git a/node/core/pvf/src/prepare/worker.rs b/node/core/pvf/src/prepare/worker.rs index 6ce8a8b91764..3de036f38d25 100644 --- a/node/core/pvf/src/prepare/worker.rs +++ b/node/core/pvf/src/prepare/worker.rs @@ -37,6 +37,8 @@ use std::{sync::Arc, time::Duration}; const NICENESS_BACKGROUND: i32 = 10; const NICENESS_FOREGROUND: i32 = 0; +/// The time period after which the preparation worker is considered unresponsive and will be killed. +// NOTE: If you change this make sure to fix the buckets of `pvf_preparation_time` metric. const COMPILATION_TIMEOUT: Duration = Duration::from_secs(60); /// Spawns a new worker with the given program path that acts as the worker and the spawn timeout.