From 6ec2c5cef7e2e76868f2e989606c7d64bb0365ed Mon Sep 17 00:00:00 2001
From: Romain Ruetschi <romain@informal.systems>
Date: Mon, 22 May 2023 13:28:09 +0200
Subject: [PATCH 01/12] Add `--debug=rpc` flag for enabling Tendermint RPC
 debug info

---
 crates/relayer-cli/src/application.rs    | 18 +++++++++++++---
 crates/relayer-cli/src/components.rs     | 25 ++++++++++++++++-------
 crates/relayer-cli/src/entry.rs          | 26 +++++++++++++++++++++++-
 crates/relayer/src/util.rs               |  1 +
 crates/relayer/src/util/debug_section.rs |  5 +++++
 5 files changed, 64 insertions(+), 11 deletions(-)
 create mode 100644 crates/relayer/src/util/debug_section.rs

diff --git a/crates/relayer-cli/src/application.rs b/crates/relayer-cli/src/application.rs
index 56fe481633..e46c3df6c7 100644
--- a/crates/relayer-cli/src/application.rs
+++ b/crates/relayer-cli/src/application.rs
@@ -10,7 +10,7 @@ use abscissa_core::{
     terminal::ColorChoice,
     Application, Configurable, FrameworkError, FrameworkErrorKind, StandardPaths,
 };
-use ibc_relayer::config::Config;
+use ibc_relayer::{config::Config, util::debug_section::DebugSection};
 
 use crate::{
     components::{JsonTracing, PrettyTracing},
@@ -47,6 +47,9 @@ pub struct CliApp {
     /// Toggle json output on/off. Changed with the global config option `-j` / `--json`.
     json_output: bool,
 
+    /// Enable the given debug sections.
+    debug_sections: Vec<DebugSection>,
+
     /// Path to the config file.
     config_path: Option<PathBuf>,
 }
@@ -61,6 +64,7 @@ impl Default for CliApp {
             config: CfgCell::default(),
             state: application::State::default(),
             json_output: false,
+            debug_sections: Vec::default(),
             config_path: None,
         }
     }
@@ -72,6 +76,11 @@ impl CliApp {
         self.json_output
     }
 
+    /// Returns the enabled debug sections
+    pub fn debug_sections(&self) -> &[DebugSection] {
+        &self.debug_sections
+    }
+
     /// Returns the path to the configuration file
     pub fn config_path(&self) -> Option<&PathBuf> {
         self.config_path.as_ref()
@@ -173,13 +182,16 @@ impl Application for CliApp {
         // Update the `json_output` flag used by `conclude::Output`
         self.json_output = command.json;
 
+        // Update the `debug_sections` flag
+        self.debug_sections = command.debug.iter().copied().map(Into::into).collect();
+
         if command.json {
             // Enable JSON by using the crate-level `Tracing`
-            let tracing = JsonTracing::new(config.global)?;
+            let tracing = JsonTracing::new(config.global, &self.debug_sections)?;
             Ok(vec![Box::new(terminal), Box::new(tracing)])
         } else {
             // Use abscissa's tracing, which pretty-prints to the terminal obeying log levels
-            let tracing = PrettyTracing::new(config.global)?;
+            let tracing = PrettyTracing::new(config.global, &self.debug_sections)?;
             Ok(vec![Box::new(terminal), Box::new(tracing)])
         }
     }
diff --git a/crates/relayer-cli/src/components.rs b/crates/relayer-cli/src/components.rs
index 84f5c36855..a89aa6e5a9 100644
--- a/crates/relayer-cli/src/components.rs
+++ b/crates/relayer-cli/src/components.rs
@@ -3,7 +3,10 @@
 use abscissa_core::{Component, FrameworkError, FrameworkErrorKind};
 use tracing_subscriber::{filter::EnvFilter, util::SubscriberInitExt, FmtSubscriber};
 
-use ibc_relayer::config::{GlobalConfig, LogLevel};
+use ibc_relayer::{
+    config::{GlobalConfig, LogLevel},
+    util::debug_section::DebugSection,
+};
 
 use crate::config::Error;
 
@@ -22,8 +25,8 @@ pub struct JsonTracing;
 
 impl JsonTracing {
     /// Creates a new [`JsonTracing`] component
-    pub fn new(cfg: GlobalConfig) -> Result<Self, FrameworkError> {
-        let filter = build_tracing_filter(cfg.log_level)?;
+    pub fn new(cfg: GlobalConfig, debug_sections: &[DebugSection]) -> Result<Self, FrameworkError> {
+        let filter = build_tracing_filter(cfg.log_level, debug_sections)?;
         // Note: JSON formatter is un-affected by ANSI 'color' option. Set to 'false'.
         let use_color = false;
 
@@ -55,8 +58,8 @@ pub struct PrettyTracing;
 
 impl PrettyTracing {
     /// Creates a new [`PrettyTracing`] component
-    pub fn new(cfg: GlobalConfig) -> Result<Self, FrameworkError> {
-        let filter = build_tracing_filter(cfg.log_level)?;
+    pub fn new(cfg: GlobalConfig, debug_sections: &[DebugSection]) -> Result<Self, FrameworkError> {
+        let filter = build_tracing_filter(cfg.log_level, debug_sections)?;
 
         // Construct a tracing subscriber with the supplied filter and enable reloading.
         let builder = FmtSubscriber::builder()
@@ -98,10 +101,18 @@ fn default_directive(log_level: LogLevel) -> String {
 /// Builds a tracing filter based on the input `log_level`.
 /// Enables tracing exclusively for the relayer crates.
 /// Returns error if the filter failed to build.
-fn build_tracing_filter(default_level: LogLevel) -> Result<EnvFilter, FrameworkError> {
-    let directive =
+fn build_tracing_filter(
+    default_level: LogLevel,
+    debug_sections: &[DebugSection],
+) -> Result<EnvFilter, FrameworkError> {
+    let mut directive =
         std::env::var(HERMES_LOG_VAR).unwrap_or_else(|_| default_directive(default_level));
 
+    if debug_sections.contains(&DebugSection::Rpc) {
+        // Enable debug tracing for the `tendermint_rpc` crate as well
+        directive.push_str(",tendermint_rpc=debug");
+    }
+
     // Build the filter directive
     match EnvFilter::try_new(&directive) {
         Ok(out) => Ok(out),
diff --git a/crates/relayer-cli/src/entry.rs b/crates/relayer-cli/src/entry.rs
index 51fa5afd31..67dc5d0f1b 100644
--- a/crates/relayer-cli/src/entry.rs
+++ b/crates/relayer-cli/src/entry.rs
@@ -5,11 +5,27 @@ use std::process;
 
 use abscissa_core::clap::Parser;
 use abscissa_core::{Command, Configurable, FrameworkError, Runnable};
-use clap::IntoApp;
+use clap::{IntoApp, ValueEnum};
 use ibc_relayer::config::Config;
+use ibc_relayer::util::debug_section::DebugSection;
 
 use crate::commands::CliCmd;
 
+#[derive(Copy, Clone, Debug, ValueEnum)]
+pub enum CliDebugSection {
+    Rpc,
+    // Profiling,
+}
+
+impl From<CliDebugSection> for DebugSection {
+    fn from(section: CliDebugSection) -> Self {
+        match section {
+            CliDebugSection::Rpc => DebugSection::Rpc,
+            // CliDebugSection::Profiling => DebugSection::Profiling,
+        }
+    }
+}
+
 /// Entry point for Hermes CLI.
 #[derive(Command, Debug, Parser)]
 #[clap(author, about, version)]
@@ -22,6 +38,14 @@ pub struct EntryPoint {
     #[clap(long = "json", help = "Enable JSON output")]
     pub json: bool,
 
+    /// Enable the given debug sections, separated by commas.
+    #[clap(
+        long = "debug",
+        help = "Enable debug output for the given section, can be repeated.",
+        value_enum
+    )]
+    pub debug: Vec<CliDebugSection>,
+
     /// Subcommand to execute.
     ///
     /// The `command` option will delegate option parsing to the command type,
diff --git a/crates/relayer/src/util.rs b/crates/relayer/src/util.rs
index e33fe52b57..997a1bc781 100644
--- a/crates/relayer/src/util.rs
+++ b/crates/relayer/src/util.rs
@@ -2,6 +2,7 @@ mod block_on;
 pub use block_on::{block_on, spawn_blocking};
 
 pub mod collate;
+pub mod debug_section;
 pub mod diff;
 pub mod iter;
 pub mod lock;
diff --git a/crates/relayer/src/util/debug_section.rs b/crates/relayer/src/util/debug_section.rs
new file mode 100644
index 0000000000..b6eef7ff4d
--- /dev/null
+++ b/crates/relayer/src/util/debug_section.rs
@@ -0,0 +1,5 @@
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum DebugSection {
+    Rpc,
+    // Profiling,
+}

From 2938d8bb29ece831980145dc1cd7a318d9376cae Mon Sep 17 00:00:00 2001
From: Romain Ruetschi <romain@informal.systems>
Date: Mon, 22 May 2023 13:31:22 +0200
Subject: [PATCH 02/12] Add `--debug=rpc` flag for enabling profiling output

---
 crates/relayer-cli/Cargo.toml            |  1 -
 crates/relayer-cli/src/commands/start.rs | 10 ++++++----
 crates/relayer-cli/src/entry.rs          |  4 ++--
 crates/relayer/Cargo.toml                |  1 -
 crates/relayer/src/macros.rs             |  5 +----
 crates/relayer/src/util/debug_section.rs |  2 +-
 6 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/crates/relayer-cli/Cargo.toml b/crates/relayer-cli/Cargo.toml
index 2441cdf03b..bd030efacf 100644
--- a/crates/relayer-cli/Cargo.toml
+++ b/crates/relayer-cli/Cargo.toml
@@ -21,7 +21,6 @@ name = "hermes"
 default     = ["telemetry", "rest-server", "std", "eyre_tracer",]
 std         = ["flex-error/std"]
 eyre_tracer = ["flex-error/eyre_tracer"]
-profiling   = ["ibc-relayer/profiling"]
 telemetry   = ["ibc-relayer/telemetry", "ibc-telemetry"]
 rest-server = ["ibc-relayer-rest"]
 
diff --git a/crates/relayer-cli/src/commands/start.rs b/crates/relayer-cli/src/commands/start.rs
index 3d9d3a4dfd..f7aba1047d 100644
--- a/crates/relayer-cli/src/commands/start.rs
+++ b/crates/relayer-cli/src/commands/start.rs
@@ -1,4 +1,5 @@
 use ibc_relayer::supervisor::SupervisorOptions;
+use ibc_relayer::util::debug_section::DebugSection;
 use std::error::Error;
 use std::io;
 
@@ -27,8 +28,9 @@ pub struct StartCmd {
 
 impl Runnable for StartCmd {
     fn run(&self) {
-        #[cfg(feature = "profiling")]
-        {
+        let app = app_reader();
+
+        if app.debug_sections().contains(&DebugSection::Profiling) {
             use chrono::prelude::*;
             use std::env;
             use std::path::Path;
@@ -46,8 +48,8 @@ impl Runnable for StartCmd {
                 now.minute(),
                 now.second()
             );
-            let file_name = Path::new(&path_str);
-            ibc_relayer::macros::profiling::open_or_create_profile_file(file_name);
+
+            ibc_relayer::macros::profiling::open_or_create_profile_file(Path::new(&path_str));
         }
 
         let config = (*app_config()).clone();
diff --git a/crates/relayer-cli/src/entry.rs b/crates/relayer-cli/src/entry.rs
index 67dc5d0f1b..f2691c40fe 100644
--- a/crates/relayer-cli/src/entry.rs
+++ b/crates/relayer-cli/src/entry.rs
@@ -14,14 +14,14 @@ use crate::commands::CliCmd;
 #[derive(Copy, Clone, Debug, ValueEnum)]
 pub enum CliDebugSection {
     Rpc,
-    // Profiling,
+    Profiling,
 }
 
 impl From<CliDebugSection> for DebugSection {
     fn from(section: CliDebugSection) -> Self {
         match section {
             CliDebugSection::Rpc => DebugSection::Rpc,
-            // CliDebugSection::Profiling => DebugSection::Profiling,
+            CliDebugSection::Profiling => DebugSection::Profiling,
         }
     }
 }
diff --git a/crates/relayer/Cargo.toml b/crates/relayer/Cargo.toml
index f4644295dc..eb096d0c46 100644
--- a/crates/relayer/Cargo.toml
+++ b/crates/relayer/Cargo.toml
@@ -17,7 +17,6 @@ all-features = true
 
 [features]
 default   = ["flex-error/std", "flex-error/eyre_tracer"]
-profiling = []
 telemetry = ["ibc-telemetry"]
 
 [dependencies]
diff --git a/crates/relayer/src/macros.rs b/crates/relayer/src/macros.rs
index f996a9b776..0b7eb0232d 100644
--- a/crates/relayer/src/macros.rs
+++ b/crates/relayer/src/macros.rs
@@ -1,4 +1,3 @@
-#[cfg(feature = "profiling")]
 pub mod profiling {
 
     use core::sync::atomic::AtomicUsize;
@@ -93,7 +92,7 @@ pub mod profiling {
 
 /// Measure the time until the current scope ends.
 ///
-/// Only enabled when the "profiling" feature is enabled.
+/// Only enabled when "--debug=profiling" is set.
 ///
 /// ## Example
 ///
@@ -115,12 +114,10 @@ pub mod profiling {
 #[macro_export]
 macro_rules! time {
     ($name:expr, $info:tt) => {
-        #[cfg(feature = "profiling")]
         let _timer = $crate::macros::profiling::Timer::new($name, ::serde_json::json!($info));
     };
 
     ($name:expr) => {
-        #[cfg(feature = "profiling")]
         let _timer = $crate::macros::profiling::Timer::new($name, ::serde_json::json!({}));
     };
 }
diff --git a/crates/relayer/src/util/debug_section.rs b/crates/relayer/src/util/debug_section.rs
index b6eef7ff4d..af3a274659 100644
--- a/crates/relayer/src/util/debug_section.rs
+++ b/crates/relayer/src/util/debug_section.rs
@@ -1,5 +1,5 @@
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
 pub enum DebugSection {
     Rpc,
-    // Profiling,
+    Profiling,
 }

From f61aa8ae180c149d38a4f473cd29cbc0de77e2a8 Mon Sep 17 00:00:00 2001
From: Romain Ruetschi <romain@informal.systems>
Date: Mon, 22 May 2023 15:14:02 +0200
Subject: [PATCH 03/12] Only output profiling info on console if profiling is
 enabled

---
 crates/relayer-cli/src/application.rs    |  10 ++
 crates/relayer-cli/src/commands/start.rs |   6 +-
 crates/relayer/src/lib.rs                |   1 -
 crates/relayer/src/macros.rs             | 123 --------------------
 crates/relayer/src/util.rs               |   1 +
 crates/relayer/src/util/profiling.rs     | 137 +++++++++++++++++++++++
 6 files changed, 152 insertions(+), 126 deletions(-)
 delete mode 100644 crates/relayer/src/macros.rs
 create mode 100644 crates/relayer/src/util/profiling.rs

diff --git a/crates/relayer-cli/src/application.rs b/crates/relayer-cli/src/application.rs
index e46c3df6c7..a40db14b6a 100644
--- a/crates/relayer-cli/src/application.rs
+++ b/crates/relayer-cli/src/application.rs
@@ -81,6 +81,11 @@ impl CliApp {
         &self.debug_sections
     }
 
+    /// Returns `true` if the given debug section is enabled
+    pub fn debug_enabled(&self, section: DebugSection) -> bool {
+        self.debug_sections.contains(&section)
+    }
+
     /// Returns the path to the configuration file
     pub fn config_path(&self) -> Option<&PathBuf> {
         self.config_path.as_ref()
@@ -185,6 +190,11 @@ impl Application for CliApp {
         // Update the `debug_sections` flag
         self.debug_sections = command.debug.iter().copied().map(Into::into).collect();
 
+        // Enable profiling if requested
+        if self.debug_enabled(DebugSection::Profiling) {
+            ibc_relayer::util::profiling::enable();
+        }
+
         if command.json {
             // Enable JSON by using the crate-level `Tracing`
             let tracing = JsonTracing::new(config.global, &self.debug_sections)?;
diff --git a/crates/relayer-cli/src/commands/start.rs b/crates/relayer-cli/src/commands/start.rs
index f7aba1047d..5b1e8d0d2d 100644
--- a/crates/relayer-cli/src/commands/start.rs
+++ b/crates/relayer-cli/src/commands/start.rs
@@ -30,11 +30,13 @@ impl Runnable for StartCmd {
     fn run(&self) {
         let app = app_reader();
 
-        if app.debug_sections().contains(&DebugSection::Profiling) {
+        if app.debug_enabled(DebugSection::Profiling) {
             use chrono::prelude::*;
             use std::env;
             use std::path::Path;
 
+            use ibc_relayer::util::profiling::open_or_create_profile_file;
+
             let profile_dir = env::var("PROFILE_DIR").unwrap_or_else(|_| ".".to_string());
 
             let now = Utc::now();
@@ -49,7 +51,7 @@ impl Runnable for StartCmd {
                 now.second()
             );
 
-            ibc_relayer::macros::profiling::open_or_create_profile_file(Path::new(&path_str));
+            open_or_create_profile_file(Path::new(&path_str));
         }
 
         let config = (*app_config()).clone();
diff --git a/crates/relayer/src/lib.rs b/crates/relayer/src/lib.rs
index 17668e69ea..8ef35e6629 100644
--- a/crates/relayer/src/lib.rs
+++ b/crates/relayer/src/lib.rs
@@ -36,7 +36,6 @@ pub mod foreign_client;
 pub mod keyring;
 pub mod light_client;
 pub mod link;
-pub mod macros;
 pub mod misbehaviour;
 pub mod object;
 pub mod path;
diff --git a/crates/relayer/src/macros.rs b/crates/relayer/src/macros.rs
deleted file mode 100644
index 0b7eb0232d..0000000000
--- a/crates/relayer/src/macros.rs
+++ /dev/null
@@ -1,123 +0,0 @@
-pub mod profiling {
-
-    use core::sync::atomic::AtomicUsize;
-    use core::sync::atomic::Ordering::Relaxed;
-    use once_cell::sync::OnceCell;
-    use serde_derive::Serialize;
-    use serde_json::Value;
-    use std::fs::File;
-    use std::fs::OpenOptions;
-    use std::path::Path;
-
-    std::thread_local! {
-        pub static DEPTH: AtomicUsize = AtomicUsize::new(0);
-    }
-    static FILE: OnceCell<File> = OnceCell::new();
-
-    /// Measure the time between when this value is allocated
-    /// and when it is dropped.
-    pub struct Timer {
-        name: &'static str,
-        info: Value,
-        start: std::time::Instant,
-    }
-
-    #[derive(Debug, Serialize)]
-    struct TimerInfo<'a> {
-        name: &'a str,
-        #[serde(flatten)]
-        info: &'a Value,
-        elapsed: u128,
-    }
-
-    impl Timer {
-        pub fn new(name: &'static str, info: Value) -> Self {
-            let depth = DEPTH.with(|d| d.fetch_add(1, Relaxed));
-            let pad = "   ".repeat(depth);
-
-            tracing::info!("{}⏳ {} - start", pad, name);
-
-            Self {
-                name,
-                info,
-                start: std::time::Instant::now(),
-            }
-        }
-    }
-
-    impl Drop for Timer {
-        fn drop(&mut self) {
-            let elapsed = self.start.elapsed().as_millis();
-
-            let depth = DEPTH.with(|d| d.fetch_sub(1, Relaxed));
-            let pad = "   ".repeat(depth - 1);
-
-            tracing::info!("{}⏳ {} - elapsed: {}ms", pad, self.name, elapsed);
-
-            let info = TimerInfo {
-                name: self.name,
-                info: &self.info,
-                elapsed,
-            };
-
-            to_json_format(&info);
-        }
-    }
-
-    pub fn open_or_create_profile_file(file_name: &Path) {
-        let file = OpenOptions::new()
-            .write(true)
-            .append(true)
-            .create(true)
-            .open(file_name)
-            .unwrap();
-
-        match FILE.set(file) {
-            Ok(_) => tracing::trace!("profile file created at: {file_name:#?}"),
-            Err(e) => tracing::error!("profile file was already set with: {:#?}", e.metadata()),
-        }
-    }
-
-    fn to_json_format(info: &TimerInfo<'_>) {
-        match FILE.get() {
-            Some(mut f) => {
-                if let Err(e) = serde_json::to_writer(&mut f, &info) {
-                    tracing::error!("Couldn't write to file: {e}");
-                }
-            }
-            None => tracing::debug!("File for profiling is not set"),
-        }
-    }
-}
-
-/// Measure the time until the current scope ends.
-///
-/// Only enabled when "--debug=profiling" is set.
-///
-/// ## Example
-///
-/// ```rust
-/// use ibc_relayer::time;
-///
-/// let val1 = 1;
-/// let val2 = 2;
-/// time!("full scope", {"id1": val1, "id2": val2});
-///
-/// let x = {
-///   time!("inner scope", {});
-///
-///   42
-///   // "inner scope" timer ends here
-/// };
-/// // "full scope" timer ends here
-/// ```
-#[macro_export]
-macro_rules! time {
-    ($name:expr, $info:tt) => {
-        let _timer = $crate::macros::profiling::Timer::new($name, ::serde_json::json!($info));
-    };
-
-    ($name:expr) => {
-        let _timer = $crate::macros::profiling::Timer::new($name, ::serde_json::json!({}));
-    };
-}
diff --git a/crates/relayer/src/util.rs b/crates/relayer/src/util.rs
index 997a1bc781..589b2f5ff4 100644
--- a/crates/relayer/src/util.rs
+++ b/crates/relayer/src/util.rs
@@ -7,6 +7,7 @@ pub mod diff;
 pub mod iter;
 pub mod lock;
 pub mod pretty;
+pub mod profiling;
 pub mod queue;
 pub mod retry;
 pub mod stream;
diff --git a/crates/relayer/src/util/profiling.rs b/crates/relayer/src/util/profiling.rs
new file mode 100644
index 0000000000..b562a9624c
--- /dev/null
+++ b/crates/relayer/src/util/profiling.rs
@@ -0,0 +1,137 @@
+use core::sync::atomic::AtomicUsize;
+use core::sync::atomic::Ordering::Relaxed;
+use once_cell::sync::OnceCell;
+use serde_derive::Serialize;
+use serde_json::Value;
+use std::fs::File;
+use std::fs::OpenOptions;
+use std::path::Path;
+
+std::thread_local! {
+    pub static DEPTH: AtomicUsize = AtomicUsize::new(0);
+}
+
+static FILE: OnceCell<File> = OnceCell::new();
+static ENABLED: OnceCell<bool> = OnceCell::new();
+
+pub fn enable() {
+    ENABLED.set(true).expect("profiling already enabled");
+}
+
+pub fn enabled() -> bool {
+    ENABLED.get().copied().unwrap_or(false)
+}
+
+/// Measure the time between when this value is allocated
+/// and when it is dropped.
+pub struct Timer {
+    name: &'static str,
+    info: Value,
+    start: std::time::Instant,
+}
+
+#[derive(Debug, Serialize)]
+struct TimerInfo<'a> {
+    name: &'a str,
+    #[serde(flatten)]
+    info: &'a Value,
+    elapsed: u128,
+}
+
+impl Timer {
+    pub fn new(name: &'static str, info: Value) -> Self {
+        let depth = DEPTH.with(|d| d.fetch_add(1, Relaxed));
+        let pad = "   ".repeat(depth);
+
+        if enabled() {
+            tracing::info!("{}⏳ {} - start", pad, name);
+        }
+
+        Self {
+            name,
+            info,
+            start: std::time::Instant::now(),
+        }
+    }
+}
+
+impl Drop for Timer {
+    fn drop(&mut self) {
+        let enabled = enabled();
+
+        let elapsed = self.start.elapsed().as_millis();
+
+        let depth = DEPTH.with(|d| d.fetch_sub(1, Relaxed));
+        let pad = "   ".repeat(depth - 1);
+
+        if enabled {
+            tracing::info!("{}⏳ {} - elapsed: {}ms", pad, self.name, elapsed);
+        }
+
+        let info = TimerInfo {
+            name: self.name,
+            info: &self.info,
+            elapsed,
+        };
+
+        if enabled {
+            output_json(&info);
+        }
+    }
+}
+
+pub fn open_or_create_profile_file(file_name: &Path) {
+    let file = OpenOptions::new()
+        .write(true)
+        .append(true)
+        .create(true)
+        .open(file_name)
+        .unwrap();
+
+    match FILE.set(file) {
+        Ok(_) => tracing::trace!("profile file created at: {file_name:#?}"),
+        Err(e) => tracing::error!("profile file was already set with: {:#?}", e.metadata()),
+    }
+}
+
+fn output_json(info: &TimerInfo<'_>) {
+    if let Some(mut f) = FILE.get() {
+        if let Err(e) = serde_json::to_writer(&mut f, &info) {
+            tracing::error!("couldn't write to file: {e}");
+        }
+    } else {
+        tracing::debug!("File for profiling output is not set");
+    }
+}
+
+/// Measure the time until the current scope ends.
+///
+/// Only enabled when "--debug=profiling" is set.
+///
+/// ## Example
+///
+/// ```rust
+/// use ibc_relayer::time;
+///
+/// let val1 = 1;
+/// let val2 = 2;
+/// time!("full scope", {"id1": val1, "id2": val2});
+///
+/// let x = {
+///   time!("inner scope", {});
+///
+///   42
+///   // "inner scope" timer ends here
+/// };
+/// // "full scope" timer ends here
+/// ```
+#[macro_export]
+macro_rules! time {
+    ($name:expr, $info:tt) => {
+        let _timer = $crate::util::profiling::Timer::new($name, ::serde_json::json!($info));
+    };
+
+    ($name:expr) => {
+        let _timer = $crate::util::profiling::Timer::new($name, ::serde_json::json!({}));
+    };
+}

From 4c75120ea93ea75c88da4ab383487dbf54545df1 Mon Sep 17 00:00:00 2001
From: Romain Ruetschi <romain@informal.systems>
Date: Mon, 22 May 2023 15:22:33 +0200
Subject: [PATCH 04/12] Allow comma-separated values

---
 crates/relayer-cli/src/entry.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/crates/relayer-cli/src/entry.rs b/crates/relayer-cli/src/entry.rs
index f2691c40fe..22ecc48622 100644
--- a/crates/relayer-cli/src/entry.rs
+++ b/crates/relayer-cli/src/entry.rs
@@ -41,8 +41,9 @@ pub struct EntryPoint {
     /// Enable the given debug sections, separated by commas.
     #[clap(
         long = "debug",
-        help = "Enable debug output for the given section, can be repeated.",
-        value_enum
+        help = "Enable debug output for the given section(s), comma separated, can be repeated.",
+        value_enum,
+        value_delimiter = ','
     )]
     pub debug: Vec<CliDebugSection>,
 

From 42a78f8e4f0fb827bd6bbd9fcf4c9f65c0232d3b Mon Sep 17 00:00:00 2001
From: Romain Ruetschi <romain@informal.systems>
Date: Mon, 22 May 2023 15:57:19 +0200
Subject: [PATCH 05/12] Re-generate templates

---
 guide/src/templates/help_templates/help.md | 2 ++
 scripts/auto_gen_templates.sh              | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/guide/src/templates/help_templates/help.md b/guide/src/templates/help_templates/help.md
index 6c56d6d60d..6cfe7e9e97 100644
--- a/guide/src/templates/help_templates/help.md
+++ b/guide/src/templates/help_templates/help.md
@@ -7,6 +7,8 @@ USAGE:
 
 OPTIONS:
         --config <CONFIG>    Path to configuration file
+        --debug <DEBUG>      Enable debug output for the given section(s), comma separated, can be
+                             repeated. [possible values: rpc, profiling]
     -h, --help               Print help information
         --json               Enable JSON output
     -V, --version            Print version information
diff --git a/scripts/auto_gen_templates.sh b/scripts/auto_gen_templates.sh
index 5ad8363f5e..d7eb557357 100644
--- a/scripts/auto_gen_templates.sh
+++ b/scripts/auto_gen_templates.sh
@@ -185,6 +185,8 @@ function generate_templates(){
             local dir="${tmp%/*}"
             mkdir -p $dir
 
+            echo "Generating template for command '$command'..."
+
             local cpt=1
             cargo run -q --bin hermes $command | $SED -n '/USAGE:/, /OPTIONS:/{ /USAGE:/! { /OPTIONS:/! p }}'  | $SED -r '/^\s*$/d ; s/^\s+// ; s/</[[#/g ; s/>/]]/g; s/hermes/[[#BINARY hermes]][[#GLOBALOPTIONS]]/ ; s/ \[(OPTIONS|SUBCOMMAND)]/\[\[#\1]]/g ;' | while read line || [[ -n $line ]]
             do

From 1592e168ffc9309bf237e392312fad8b62f3623b Mon Sep 17 00:00:00 2001
From: Romain Ruetschi <romain@informal.systems>
Date: Mon, 22 May 2023 16:37:47 +0200
Subject: [PATCH 06/12] Add `json-profiling` debug section, change env variable
 to `PROFILING_DIR`

---
 crates/relayer-cli/src/application.rs         |  3 +-
 crates/relayer-cli/src/commands/start.rs      |  6 +--
 crates/relayer-cli/src/entry.rs               |  2 +
 crates/relayer/src/util/debug_section.rs      |  1 +
 crates/relayer/src/util/profiling.rs          | 29 +++++++-----
 .../src/advanced/troubleshooting/profiling.md | 44 +++++++++++--------
 6 files changed, 51 insertions(+), 34 deletions(-)

diff --git a/crates/relayer-cli/src/application.rs b/crates/relayer-cli/src/application.rs
index a40db14b6a..c4f9729d46 100644
--- a/crates/relayer-cli/src/application.rs
+++ b/crates/relayer-cli/src/application.rs
@@ -192,7 +192,8 @@ impl Application for CliApp {
 
         // Enable profiling if requested
         if self.debug_enabled(DebugSection::Profiling) {
-            ibc_relayer::util::profiling::enable();
+            let enable_json = self.debug_enabled(DebugSection::ProfilingJson);
+            ibc_relayer::util::profiling::enable(enable_json);
         }
 
         if command.json {
diff --git a/crates/relayer-cli/src/commands/start.rs b/crates/relayer-cli/src/commands/start.rs
index 5b1e8d0d2d..e75685cd8c 100644
--- a/crates/relayer-cli/src/commands/start.rs
+++ b/crates/relayer-cli/src/commands/start.rs
@@ -30,18 +30,18 @@ impl Runnable for StartCmd {
     fn run(&self) {
         let app = app_reader();
 
-        if app.debug_enabled(DebugSection::Profiling) {
+        if app.debug_enabled(DebugSection::ProfilingJson) {
             use chrono::prelude::*;
             use std::env;
             use std::path::Path;
 
             use ibc_relayer::util::profiling::open_or_create_profile_file;
 
-            let profile_dir = env::var("PROFILE_DIR").unwrap_or_else(|_| ".".to_string());
+            let profile_dir = env::var("PROFILING_DIR").unwrap_or_else(|_| ".".to_string());
 
             let now = Utc::now();
             let path_str = format!(
-                "{}/hermes-{:04}-{:02}-{:02}-{:02}{:02}{:02}.prof",
+                "{}/hermes-{:04}-{:02}-{:02}-{:02}{:02}{:02}-prof.json",
                 profile_dir,
                 now.year(),
                 now.month(),
diff --git a/crates/relayer-cli/src/entry.rs b/crates/relayer-cli/src/entry.rs
index 22ecc48622..29fa21c0da 100644
--- a/crates/relayer-cli/src/entry.rs
+++ b/crates/relayer-cli/src/entry.rs
@@ -15,6 +15,7 @@ use crate::commands::CliCmd;
 pub enum CliDebugSection {
     Rpc,
     Profiling,
+    ProfilingJson,
 }
 
 impl From<CliDebugSection> for DebugSection {
@@ -22,6 +23,7 @@ impl From<CliDebugSection> for DebugSection {
         match section {
             CliDebugSection::Rpc => DebugSection::Rpc,
             CliDebugSection::Profiling => DebugSection::Profiling,
+            CliDebugSection::ProfilingJson => DebugSection::ProfilingJson,
         }
     }
 }
diff --git a/crates/relayer/src/util/debug_section.rs b/crates/relayer/src/util/debug_section.rs
index af3a274659..79fbaef5b6 100644
--- a/crates/relayer/src/util/debug_section.rs
+++ b/crates/relayer/src/util/debug_section.rs
@@ -2,4 +2,5 @@
 pub enum DebugSection {
     Rpc,
     Profiling,
+    ProfilingJson,
 }
diff --git a/crates/relayer/src/util/profiling.rs b/crates/relayer/src/util/profiling.rs
index b562a9624c..d2416889a2 100644
--- a/crates/relayer/src/util/profiling.rs
+++ b/crates/relayer/src/util/profiling.rs
@@ -1,27 +1,36 @@
-use core::sync::atomic::AtomicUsize;
-use core::sync::atomic::Ordering::Relaxed;
+use core::sync::atomic::{AtomicUsize, Ordering::Relaxed};
+use std::fs::{File, OpenOptions};
+use std::path::Path;
+use std::sync::Mutex;
+
 use once_cell::sync::OnceCell;
 use serde_derive::Serialize;
 use serde_json::Value;
-use std::fs::File;
-use std::fs::OpenOptions;
-use std::path::Path;
 
 std::thread_local! {
     pub static DEPTH: AtomicUsize = AtomicUsize::new(0);
 }
 
-static FILE: OnceCell<File> = OnceCell::new();
+static FILE: OnceCell<Mutex<File>> = OnceCell::new();
 static ENABLED: OnceCell<bool> = OnceCell::new();
+static JSON_ENABLED: OnceCell<bool> = OnceCell::new();
 
-pub fn enable() {
+pub fn enable(enable_json: bool) {
     ENABLED.set(true).expect("profiling already enabled");
+
+    JSON_ENABLED
+        .set(enable_json)
+        .expect("JSON profiling already enabled");
 }
 
 pub fn enabled() -> bool {
     ENABLED.get().copied().unwrap_or(false)
 }
 
+pub fn json_enabled() -> bool {
+    JSON_ENABLED.get().copied().unwrap_or(false)
+}
+
 /// Measure the time between when this value is allocated
 /// and when it is dropped.
 pub struct Timer {
@@ -57,14 +66,12 @@ impl Timer {
 
 impl Drop for Timer {
     fn drop(&mut self) {
-        let enabled = enabled();
-
         let elapsed = self.start.elapsed().as_millis();
 
         let depth = DEPTH.with(|d| d.fetch_sub(1, Relaxed));
         let pad = "   ".repeat(depth - 1);
 
-        if enabled {
+        if enabled() {
             tracing::info!("{}⏳ {} - elapsed: {}ms", pad, self.name, elapsed);
         }
 
@@ -74,7 +81,7 @@ impl Drop for Timer {
             elapsed,
         };
 
-        if enabled {
+        if json_enabled() {
             output_json(&info);
         }
     }
diff --git a/guide/src/advanced/troubleshooting/profiling.md b/guide/src/advanced/troubleshooting/profiling.md
index 956011bbdd..bc952aa8a7 100644
--- a/guide/src/advanced/troubleshooting/profiling.md
+++ b/guide/src/advanced/troubleshooting/profiling.md
@@ -4,25 +4,12 @@ The `relayer` crate provides a `time!` macro which can be used to measure how mu
 
 ### Setup
 
-The `time!` macro has no effect unless the `profiling` feature of the `relayer` crate is enabled.
-
-To enable it, one must compile the `relayer-cli` crate with the `--features=profiling` flag.
-
-a) One way is to build the `relayer` binary and update the `hermes` alias to point to the executable:
+The `time!` macro has no effect unless the `--debug=profiling` global flag is specified on the command-line:
 
 ```shell
-cd relayer-cli/
-cargo build --features=profiling
+$ hermes --debug=profiling start
 ```
 
-b) Alternatively, one can use the `cargo run` command and update the alias accordingly:
-
-```shell
-alias hermes='cargo run --features=profiling --manifest-path=relayer-cli/Cargo.toml --'
-```
-
-The `--manifest-path=relayer-cli/Cargo.toml` flag is needed for `cargo run` to accept the `--features` flag.
-
 ### Example
 
 ```rust
@@ -45,7 +32,7 @@ fn my_function(x: u32) -> u32 {
 }
 ```
 
-#### Output
+#### Console output
 
 ```
 Jan 20 11:28:46.841  INFO relayer::macros::profiling: ⏳ myfunction: x=42 - start
@@ -60,10 +47,9 @@ and it's often not clear which of these are the culprit for low performance.
 With profiling enabled, `hermes` will output timing information for individual
 methods involved in a command.
 
-__NOTE__: To be able to see the profiling output, Hermes needs to be compiled with
-the `profiling` feature and the [log level][log-level] should be `info` level or lower.
+__NOTE__: To be able to see the profiling output, the [log level][log-level] should be `info` level or lower.
 
-#### Example output for `tx conn-init` command
+##### Example output for `tx conn-init` command
 
 ```
 {{#template ../../templates/commands/hermes/tx/conn-init_1.md DST_CHAIN_ID=ibc-0 SRC_CHAIN_ID=ibc-1 DST_CLIENT_ID=07-tendermint-0 SRC_CLIENT_ID=07-tendermint-0}}
@@ -123,3 +109,23 @@ Success: CreateClient(
     ),
 )
 ```
+
+#### JSON output
+
+Additionally, if the `--debug=profiling-json` flag is specified, Hermes will output profiling information in
+JSON format in a file named `hermes-YYYY-MM-DD-HHMMSS-prof.json`, in the directory specified in the `PROFILING_DIR`
+env variable, or the current directory otherwise.
+
+```json
+{"name":"fetch_node_info","src_chain":"ibc-0","elapsed":6}
+{"name":"chain_status","src_chain":"ibc-0","elapsed":12}
+{"name":"query_config_params","src_chain":"ibc-0","elapsed":3}
+{"name":"min_gas_price","src_chain":"ibc-0","elapsed":3}
+{"name":"query_staking_params","src_chain":"ibc-0","elapsed":159}
+{"name":"historical_entries","src_chain":"ibc-0","elapsed":329}
+{"name":"query_staking_params","src_chain":"ibc-0","elapsed":121}
+{"name":"unbonding_period","src_chain":"ibc-0","elapsed":12}
+{"name":"query_latest_height","src_chain":"ibc-0","elapsed":8}
+{"name":"fetch_node_info","src_chain":"ibc-1","elapsed":9}
+{"name":"chain_status","src_chain":"ibc-1","elapsed":43}
+```

From d0953044c9920204e180410f6631e328ef6f2a58 Mon Sep 17 00:00:00 2001
From: Romain Ruetschi <romain@informal.systems>
Date: Mon, 22 May 2023 16:38:32 +0200
Subject: [PATCH 07/12] Add newline after each JSON entry

This requires a mutex around the File, otherwise we may end up
with garbled entries when two threads write the file concurrently.
---
 crates/relayer/src/util/profiling.rs | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/crates/relayer/src/util/profiling.rs b/crates/relayer/src/util/profiling.rs
index d2416889a2..e058e82116 100644
--- a/crates/relayer/src/util/profiling.rs
+++ b/crates/relayer/src/util/profiling.rs
@@ -95,15 +95,18 @@ pub fn open_or_create_profile_file(file_name: &Path) {
         .open(file_name)
         .unwrap();
 
-    match FILE.set(file) {
+    match FILE.set(Mutex::new(file)) {
         Ok(_) => tracing::trace!("profile file created at: {file_name:#?}"),
-        Err(e) => tracing::error!("profile file was already set with: {:#?}", e.metadata()),
+        Err(e) => tracing::error!(
+            "profile file was already set with: {:#?}",
+            e.lock().unwrap().metadata()
+        ),
     }
 }
 
 fn output_json(info: &TimerInfo<'_>) {
-    if let Some(mut f) = FILE.get() {
-        if let Err(e) = serde_json::to_writer(&mut f, &info) {
+    if let Some(f) = FILE.get() {
+        if let Err(e) = _output_json(f, info) {
             tracing::error!("couldn't write to file: {e}");
         }
     } else {
@@ -111,6 +114,18 @@ fn output_json(info: &TimerInfo<'_>) {
     }
 }
 
+fn _output_json(f: &Mutex<File>, info: &TimerInfo<'_>) -> Result<(), Box<dyn std::error::Error>> {
+    use std::io::Write;
+
+    {
+        let mut f = f.lock().unwrap();
+        serde_json::to_writer(&mut *f, info)?;
+        writeln!(&mut *f)?;
+    }
+
+    Ok(())
+}
+
 /// Measure the time until the current scope ends.
 ///
 /// Only enabled when "--debug=profiling" is set.

From eacd400a7154c2ed4cac726e1c97876b96cb1155 Mon Sep 17 00:00:00 2001
From: Romain Ruetschi <romain@informal.systems>
Date: Mon, 22 May 2023 16:53:51 +0200
Subject: [PATCH 08/12] Use atomics instead of once cell for profiling flags

---
 crates/relayer-cli/src/application.rs |  7 +++---
 crates/relayer/src/util/profiling.rs  | 35 +++++++++++++--------------
 2 files changed, 20 insertions(+), 22 deletions(-)

diff --git a/crates/relayer-cli/src/application.rs b/crates/relayer-cli/src/application.rs
index c4f9729d46..b2bca10c0a 100644
--- a/crates/relayer-cli/src/application.rs
+++ b/crates/relayer-cli/src/application.rs
@@ -191,10 +191,9 @@ impl Application for CliApp {
         self.debug_sections = command.debug.iter().copied().map(Into::into).collect();
 
         // Enable profiling if requested
-        if self.debug_enabled(DebugSection::Profiling) {
-            let enable_json = self.debug_enabled(DebugSection::ProfilingJson);
-            ibc_relayer::util::profiling::enable(enable_json);
-        }
+        let enable_console = self.debug_enabled(DebugSection::Profiling);
+        let enable_json = self.debug_enabled(DebugSection::ProfilingJson);
+        ibc_relayer::util::profiling::enable(enable_console, enable_json);
 
         if command.json {
             // Enable JSON by using the crate-level `Tracing`
diff --git a/crates/relayer/src/util/profiling.rs b/crates/relayer/src/util/profiling.rs
index e058e82116..e0404d9695 100644
--- a/crates/relayer/src/util/profiling.rs
+++ b/crates/relayer/src/util/profiling.rs
@@ -1,6 +1,6 @@
-use core::sync::atomic::{AtomicUsize, Ordering::Relaxed};
 use std::fs::{File, OpenOptions};
 use std::path::Path;
+use std::sync::atomic::{AtomicUsize, Ordering::Relaxed};
 use std::sync::Mutex;
 
 use once_cell::sync::OnceCell;
@@ -12,23 +12,24 @@ std::thread_local! {
 }
 
 static FILE: OnceCell<Mutex<File>> = OnceCell::new();
-static ENABLED: OnceCell<bool> = OnceCell::new();
-static JSON_ENABLED: OnceCell<bool> = OnceCell::new();
+static ENABLED: AtomicUsize = AtomicUsize::new(0);
 
-pub fn enable(enable_json: bool) {
-    ENABLED.set(true).expect("profiling already enabled");
+const CONSOLE_MASK: usize = 0b01;
+const JSON_MASK: usize = 0b10;
 
-    JSON_ENABLED
-        .set(enable_json)
-        .expect("JSON profiling already enabled");
+pub fn enable(console: bool, json: bool) {
+    let console_mask = if console { CONSOLE_MASK } else { 0 };
+    let json_mask = if json { JSON_MASK } else { 0 };
+
+    ENABLED.store(console_mask | json_mask, Relaxed);
 }
 
-pub fn enabled() -> bool {
-    ENABLED.get().copied().unwrap_or(false)
+pub fn console_enabled() -> bool {
+    ENABLED.load(Relaxed) & CONSOLE_MASK != 0
 }
 
 pub fn json_enabled() -> bool {
-    JSON_ENABLED.get().copied().unwrap_or(false)
+    ENABLED.load(Relaxed) & JSON_MASK != 0
 }
 
 /// Measure the time between when this value is allocated
@@ -52,7 +53,7 @@ impl Timer {
         let depth = DEPTH.with(|d| d.fetch_add(1, Relaxed));
         let pad = "   ".repeat(depth);
 
-        if enabled() {
+        if console_enabled() {
             tracing::info!("{}⏳ {} - start", pad, name);
         }
 
@@ -71,7 +72,7 @@ impl Drop for Timer {
         let depth = DEPTH.with(|d| d.fetch_sub(1, Relaxed));
         let pad = "   ".repeat(depth - 1);
 
-        if enabled() {
+        if console_enabled() {
             tracing::info!("{}⏳ {} - elapsed: {}ms", pad, self.name, elapsed);
         }
 
@@ -117,11 +118,9 @@ fn output_json(info: &TimerInfo<'_>) {
 fn _output_json(f: &Mutex<File>, info: &TimerInfo<'_>) -> Result<(), Box<dyn std::error::Error>> {
     use std::io::Write;
 
-    {
-        let mut f = f.lock().unwrap();
-        serde_json::to_writer(&mut *f, info)?;
-        writeln!(&mut *f)?;
-    }
+    let mut f = f.lock().unwrap();
+    serde_json::to_writer(&mut *f, info)?;
+    writeln!(&mut *f)?;
 
     Ok(())
 }

From bc105a818c9a38b9b8c6bdd0f010c85fd1a6b5e2 Mon Sep 17 00:00:00 2001
From: Romain Ruetschi <romain@informal.systems>
Date: Mon, 22 May 2023 17:10:25 +0200
Subject: [PATCH 09/12] Avoid doing any work when profiling is disabled

---
 crates/relayer/src/util/profiling.rs | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/crates/relayer/src/util/profiling.rs b/crates/relayer/src/util/profiling.rs
index e0404d9695..c6310f95fc 100644
--- a/crates/relayer/src/util/profiling.rs
+++ b/crates/relayer/src/util/profiling.rs
@@ -32,6 +32,10 @@ pub fn json_enabled() -> bool {
     ENABLED.load(Relaxed) & JSON_MASK != 0
 }
 
+fn enabled() -> usize {
+    ENABLED.load(Relaxed)
+}
+
 /// Measure the time between when this value is allocated
 /// and when it is dropped.
 pub struct Timer {
@@ -67,23 +71,27 @@ impl Timer {
 
 impl Drop for Timer {
     fn drop(&mut self) {
+        let enabled = enabled();
+
+        if enabled == 0 {
+            return;
+        }
+
         let elapsed = self.start.elapsed().as_millis();
 
         let depth = DEPTH.with(|d| d.fetch_sub(1, Relaxed));
         let pad = "   ".repeat(depth - 1);
 
-        if console_enabled() {
+        if enabled & CONSOLE_MASK != 0 {
             tracing::info!("{}⏳ {} - elapsed: {}ms", pad, self.name, elapsed);
         }
 
-        let info = TimerInfo {
-            name: self.name,
-            info: &self.info,
-            elapsed,
-        };
-
-        if json_enabled() {
-            output_json(&info);
+        if enabled & JSON_MASK != 0 {
+            output_json(&TimerInfo {
+                name: self.name,
+                info: &self.info,
+                elapsed,
+            });
         }
     }
 }

From 6cfdc142750bbd205bb192eec95a12c5b1cba0b2 Mon Sep 17 00:00:00 2001
From: Romain Ruetschi <romain@informal.systems>
Date: Mon, 22 May 2023 21:56:55 +0200
Subject: [PATCH 10/12] Update templates

---
 guide/src/templates/help_templates/help.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/guide/src/templates/help_templates/help.md b/guide/src/templates/help_templates/help.md
index 6cfe7e9e97..516720fc78 100644
--- a/guide/src/templates/help_templates/help.md
+++ b/guide/src/templates/help_templates/help.md
@@ -8,7 +8,7 @@ USAGE:
 OPTIONS:
         --config <CONFIG>    Path to configuration file
         --debug <DEBUG>      Enable debug output for the given section(s), comma separated, can be
-                             repeated. [possible values: rpc, profiling]
+                             repeated. [possible values: rpc, profiling, profiling-json]
     -h, --help               Print help information
         --json               Enable JSON output
     -V, --version            Print version information

From c9e9f5a3f7bcb499e320f0636b4e7ddda15cb33f Mon Sep 17 00:00:00 2001
From: Luca Joss <luca@informal.systems>
Date: Tue, 23 May 2023 13:28:58 +0200
Subject: [PATCH 11/12] Add note to guide for profiling-json. Add changelog
 entry

---
 .../unreleased/features/ibc-relayer-cli/2852-add-debug-flag.md | 3 +++
 guide/src/advanced/troubleshooting/profiling.md                | 3 +++
 2 files changed, 6 insertions(+)
 create mode 100644 .changelog/unreleased/features/ibc-relayer-cli/2852-add-debug-flag.md

diff --git a/.changelog/unreleased/features/ibc-relayer-cli/2852-add-debug-flag.md b/.changelog/unreleased/features/ibc-relayer-cli/2852-add-debug-flag.md
new file mode 100644
index 0000000000..ec5112a7ae
--- /dev/null
+++ b/.changelog/unreleased/features/ibc-relayer-cli/2852-add-debug-flag.md
@@ -0,0 +1,3 @@
+- Add a global flag `--debug` which allows enabling
+  profiling, rpc debug logs and/or profiling to json file
+  ([#2852](https://github.com/informalsystems/hermes/issues/2852))
\ No newline at end of file
diff --git a/guide/src/advanced/troubleshooting/profiling.md b/guide/src/advanced/troubleshooting/profiling.md
index bc952aa8a7..cafc46642b 100644
--- a/guide/src/advanced/troubleshooting/profiling.md
+++ b/guide/src/advanced/troubleshooting/profiling.md
@@ -116,6 +116,9 @@ Additionally, if the `--debug=profiling-json` flag is specified, Hermes will out
 JSON format in a file named `hermes-YYYY-MM-DD-HHMMSS-prof.json`, in the directory specified in the `PROFILING_DIR`
 env variable, or the current directory otherwise.
 
+> __NOTE__: Outputting profiling information in JSON format in a file is only available for the `hermes start` command.
+> This debug option won't do anything with the other CLIs.
+
 ```json
 {"name":"fetch_node_info","src_chain":"ibc-0","elapsed":6}
 {"name":"chain_status","src_chain":"ibc-0","elapsed":12}

From fbb5f840ffe288f2562de35aa67f7a3547e46fb9 Mon Sep 17 00:00:00 2001
From: Romain Ruetschi <romain@informal.systems>
Date: Tue, 23 May 2023 13:42:11 +0200
Subject: [PATCH 12/12] Update changelog entry

Signed-off-by: Romain Ruetschi <romain.ruetschi@gmail.com>
---
 .../features/ibc-relayer-cli/2852-add-debug-flag.md         | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.changelog/unreleased/features/ibc-relayer-cli/2852-add-debug-flag.md b/.changelog/unreleased/features/ibc-relayer-cli/2852-add-debug-flag.md
index ec5112a7ae..42d86fbc0d 100644
--- a/.changelog/unreleased/features/ibc-relayer-cli/2852-add-debug-flag.md
+++ b/.changelog/unreleased/features/ibc-relayer-cli/2852-add-debug-flag.md
@@ -1,3 +1,5 @@
-- Add a global flag `--debug` which allows enabling
-  profiling, rpc debug logs and/or profiling to json file
+- Add a global flag `--debug` which can take one or more of the following values, separated by commas:
+  - `rpc`: show RPC debug logs
+  - `profiling`: show profiling information in the console
+  - `profiling-json`: dump the profiling information to a JSON file in the directory specified in `PROFILING_DIR` env variable if present, or the current directory otherwise.
   ([#2852](https://github.com/informalsystems/hermes/issues/2852))
\ No newline at end of file