From 8c0027e1bc397e2a9045132243e02692bfc900c3 Mon Sep 17 00:00:00 2001 From: Voxelot Date: Tue, 18 Apr 2023 16:56:45 -0700 Subject: [PATCH 1/6] add pyroscope to binary --- .github/workflows/ci.yml | 51 +++++++++++++++++++ bin/fuel-core/Cargo.toml | 3 ++ bin/fuel-core/src/cli/run.rs | 50 ++++++++++++++++-- bin/fuel-core/src/cli/run/profiling.rs | 10 ++++ deployment/Dockerfile | 12 +++-- .../charts/templates/fuel-core-deploy.yaml | 8 +++ deployment/charts/values.yaml | 2 + deployment/scripts/.env | 6 ++- 8 files changed, 133 insertions(+), 9 deletions(-) create mode 100644 bin/fuel-core/src/cli/run/profiling.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 804e2392ceb..088b358c318 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -284,6 +284,57 @@ jobs: cache-from: type=registry,ref=ghcr.io/fuellabs/fuel-core-build-cache:latest cache-to: type=registry,ref=ghcr.io/fuellabs/fuel-core-build-cache:latest,mode=max + # duplicate of publish-docker-image, but with profiling features enabled + # this is split into a separate action since it takes longer to build + publish-docker-image-profiling: + needs: + - verifications-complete + runs-on: buildjet-4vcpu-ubuntu-2204 + permissions: + contents: read + packages: write + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v3 + with: + images: | + ghcr.io/fuellabs/fuel-core-profiling + tags: | + type=sha + type=ref,event=branch + type=ref,event=tag + type=semver,pattern={{raw}} + flavor: | + latest=${{ github.ref == 'refs/heads/master' }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Log in to the ghcr.io registry + uses: docker/login-action@v1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push the image to ghcr.io + uses: docker/build-push-action@v2 + with: + context: . + file: deployment/Dockerfile + build-args: | + "FEATURES=production,profiling" + "DEBUG_SYMBOLS=true" + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + - uses: FuelLabs/.github/.github/actions/slack-notify-template@master if: always() && (github.ref == 'refs/heads/master' || github.ref_type == 'tag') with: diff --git a/bin/fuel-core/Cargo.toml b/bin/fuel-core/Cargo.toml index cc2fa27e4aa..a0ab5ec043c 100644 --- a/bin/fuel-core/Cargo.toml +++ b/bin/fuel-core/Cargo.toml @@ -24,6 +24,8 @@ dotenvy = { version = "0.15", optional = true } fuel-core = { workspace = true } humantime = "2.1" lazy_static = { workspace = true } +pyroscope = { version = "0.5", optional = true } +pyroscope_pprofrs = { version = "0.2", optional = true } serde_json = { workspace = true, features = ["raw_value"], optional = true } tikv-jemallocator = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } @@ -47,5 +49,6 @@ p2p = ["fuel-core/p2p", "const_format"] relayer = ["fuel-core/relayer", "dep:url", "dep:serde_json"] rocksdb = ["fuel-core/rocksdb"] rocksdb-production = ["fuel-core/rocksdb-production"] +profiling = ["dep:pyroscope", "dep:pyroscope_pprofrs"] # features to enable in production, but increase build times production = ["env", "metrics", "relayer", "rocksdb-production", "p2p"] diff --git a/bin/fuel-core/src/cli/run.rs b/bin/fuel-core/src/cli/run.rs index 6379bb0fd98..df7263ab7f9 100644 --- a/bin/fuel-core/src/cli/run.rs +++ b/bin/fuel-core/src/cli/run.rs @@ -36,6 +36,13 @@ use fuel_core::{ }, }, }; +#[cfg(feature = "profiling")] +use pyroscope::PyroscopeAgent; +#[cfg(feature = "profiling")] +use pyroscope_pprofrs::{ + pprof_backend, + PprofConfig, +}; use std::{ env, net, @@ -57,6 +64,7 @@ const DEFAULT_DATABASE_CACHE_SIZE: usize = 1024 * 1024 * 1024; mod p2p; mod consensus; +mod profiling; #[cfg(feature = "relayer")] mod relayer; @@ -193,6 +201,9 @@ pub struct Command { /// Time to wait after submitting a query before debug info will be logged about query. #[clap(long = "query-log-threshold-time", default_value = "2s", env)] pub query_log_threshold_time: humantime::Duration, + + #[clap(flatten)] + pub profiling: profiling::ProfilingArgs, } impl Command { @@ -229,6 +240,7 @@ impl Command { min_connected_reserved_peers, time_until_synced, query_log_threshold_time, + profiling: _, } = self; let addr = net::SocketAddr::new(ip, port); @@ -332,20 +344,48 @@ impl Command { } pub async fn exec(command: Command) -> anyhow::Result<()> { - let config = command.get_config()?; let network_name = { #[cfg(feature = "p2p")] { - config - .p2p - .as_ref() - .map(|config| config.network_name.clone()) + command + .p2p_args + .network + .clone() .unwrap_or_else(|| "default_network".to_string()) } #[cfg(not(feature = "p2p"))] "default_network" } .to_string(); + "default_network".to_string() + }; + + #[cfg(feature = "profiling")] + let profiling = command.profiling.clone(); + let config = command.get_config()?; + + // start profiling agent if url is configured + #[cfg(feature = "profiling")] + let _profiling_agent = profiling + .pyroscope_url + .as_ref() + .map(|url| -> anyhow::Result<_> { + // Configure profiling backend + let agent = PyroscopeAgent::builder(url, &"fuel-core".to_string()) + .tags(vec![ + ("service", config.name.as_str()), + ("network", network_name.as_str()), + ]) + .backend(pprof_backend( + PprofConfig::new().sample_rate(profiling.pprof_sample_rate), + )) + .build() + .context("failed to start profiler")?; + let agent_running = agent.start().unwrap(); + Ok(agent_running) + }) + .transpose()?; + // log fuel-core version info!("Fuel Core version v{}", env!("CARGO_PKG_VERSION")); trace!("Initializing in TRACE mode."); diff --git a/bin/fuel-core/src/cli/run/profiling.rs b/bin/fuel-core/src/cli/run/profiling.rs new file mode 100644 index 00000000000..2471e02d66e --- /dev/null +++ b/bin/fuel-core/src/cli/run/profiling.rs @@ -0,0 +1,10 @@ +use clap::Args; + +#[derive(Debug, Clone, Args)] +pub struct ProfilingArgs { + #[clap(long = "pyroscope-url", env)] + pub pyroscope_url: Option, + + #[clap(long = "pprof-sample-rate", default_value = "100", env)] + pub pprof_sample_rate: u32, +} diff --git a/deployment/Dockerfile b/deployment/Dockerfile index f4914f6de4a..3ac42306fc1 100644 --- a/deployment/Dockerfile +++ b/deployment/Dockerfile @@ -26,16 +26,22 @@ ENV CARGO_NET_GIT_FETCH_WITH_CLI=true COPY . . RUN cargo chef prepare --recipe-path recipe.json + FROM chef as builder +ARG FEATURES="production" +ARG DEBUG_SYMBOLS=false ENV CARGO_NET_GIT_FETCH_WITH_CLI=true +ENV CARGO_PROFILE_RELEASE_DEBUG=$DEBUG_SYMBOLS +ENV BUILD_FEATURES=$FEATURES COPY --from=planner /build/recipe.json recipe.json - +RUN echo $CARGO_PROFILE_RELEASE_DEBUG +RUN echo $BUILD_FEATURES # Build our project dependecies, not our application! -RUN xx-cargo chef cook --release --no-default-features --features "production" -p fuel-core-bin --recipe-path recipe.json +RUN xx-cargo chef cook --release --no-default-features --features "${BUILD_FEATURES}" -p fuel-core-bin --recipe-path recipe.json # Up to this point, if our dependency tree stays the same, # all layers should be cached. COPY . . -RUN xx-cargo build --release --no-default-features --features "production" -p fuel-core-bin \ +RUN xx-cargo build --release --no-default-features --features "$BUILD_FEATURES" -p fuel-core-bin \ && xx-verify ./target/$(xx-cargo --print-target-triple)/release/fuel-core \ && mv ./target/$(xx-cargo --print-target-triple)/release/fuel-core ./target/release/fuel-core \ && mv ./target/$(xx-cargo --print-target-triple)/release/fuel-core.d ./target/release/fuel-core.d diff --git a/deployment/charts/templates/fuel-core-deploy.yaml b/deployment/charts/templates/fuel-core-deploy.yaml index 2066b38e383..30d4da5b10c 100644 --- a/deployment/charts/templates/fuel-core-deploy.yaml +++ b/deployment/charts/templates/fuel-core-deploy.yaml @@ -238,6 +238,14 @@ spec: {{- if .Values.app.allow_private_addresses }} - "--allow-private-addresses" {{- end }} + {{- if .Values.app.pyroscope_url }} + - "--pyroscope-url" + - "{{ .Values.app.pyroscope_url }}" + {{- end }} + {{- if .Values.app.pprof_sample_rate }} + - "--pprof-sample-rate" + - "{{ .Values.app.pprof_sample_rate }}" + {{- end }} resources: limits: cpu: {{ .Values.app.resources.cpu_limits }} diff --git a/deployment/charts/values.yaml b/deployment/charts/values.yaml index 384b6932901..be30d7b8f11 100644 --- a/deployment/charts/values.yaml +++ b/deployment/charts/values.yaml @@ -38,6 +38,8 @@ app: relayer_eth_sync_call_freq_s: "${fuel_core_relayer_eth_sync_call_freq_s}" relayer_eth_sync_log_freq_s: "${fuel_core_relayer_eth_sync_log_freq_s}" sentry_enabled: ${fuel_core_sentry_enabled} + pyroscope_url: "${fuel_core_pyroscope_url}" + pprof_sample_rate: "${fuel_core_pprof_sample_rate}" image: repository: ${fuel_core_image_repository} tag: ${fuel_core_image_tag} diff --git a/deployment/scripts/.env b/deployment/scripts/.env index f225ae81bbd..7250a1144ad 100644 --- a/deployment/scripts/.env +++ b/deployment/scripts/.env @@ -30,7 +30,11 @@ fuel_core_memory_limits="100Mi" fuel_core_prometheus_enabled=true fuel_core_prometheus_helm_release_name="kube-prometheus" -# consensus key secret +# profiling agent settings +fuel_core_pyroscope_url="http://pyroscope:4040" +fuel_core_pprof_sample_rate="100" + +# consensus key secret fuel_core_consensus_key_secret="dGVzdA==" # The validator should have disabled production. It can be done by uncommenting the `fuel_core_poa_instant=false` below. From a107b2769134b227d5f7f8c5a1a434fdc9707dec Mon Sep 17 00:00:00 2001 From: Voxelot Date: Fri, 25 Aug 2023 11:46:03 -0700 Subject: [PATCH 2/6] merge fixes --- Cargo.lock | 259 +++++++++++++++++++++++++++++++++-- bin/fuel-core/src/cli/run.rs | 2 - 2 files changed, 246 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bcc63f2e506..8e34e1a0e62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -27,6 +27,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + [[package]] name = "aead" version = "0.3.2" @@ -528,6 +534,17 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1181e1e0d1fce796a03db1ae795d67167da795f9cf4a39c37589e85ef57f26d3" +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + [[package]] name = "auto_impl" version = "0.5.0" @@ -1085,6 +1102,23 @@ dependencies = [ "libloading", ] +[[package]] +name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "atty", + "bitflags 1.3.2", + "clap_derive 3.2.25", + "clap_lex 0.2.4", + "indexmap 1.9.3", + "once_cell", + "strsim", + "termcolor", + "textwrap", +] + [[package]] name = "clap" version = "4.3.24" @@ -1092,7 +1126,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb690e81c7840c0d7aade59f242ea3b41b9bc27bcd5997890e7702ae4b32e487" dependencies = [ "clap_builder", - "clap_derive", + "clap_derive 4.3.12", "once_cell", ] @@ -1104,10 +1138,23 @@ checksum = "5ed2e96bc16d8d740f6f48d663eddf4b8a0983e79210fd55479b7bcd0a69860e" dependencies = [ "anstream", "anstyle", - "clap_lex", + "clap_lex 0.5.0", "strsim", ] +[[package]] +name = "clap_derive" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "clap_derive" version = "4.3.12" @@ -1120,6 +1167,15 @@ dependencies = [ "syn 2.0.29", ] +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "clap_lex" version = "0.5.0" @@ -1384,6 +1440,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "cpp_demangle" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8227005286ec39567949b33df9896bcadfa6051bccca2488129f108ca23119" +dependencies = [ + "cfg-if", +] + [[package]] name = "cpufeatures" version = "0.2.9" @@ -1426,7 +1491,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap", + "clap 4.3.24", "criterion-plot", "futures", "is-terminal", @@ -1789,6 +1854,15 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "uuid 1.4.1", +] + [[package]] name = "der" version = "0.6.1" @@ -2559,6 +2633,18 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e825f6987101665dea6ec934c09ec6d721de7bc1bf92248e1d5810c8cd636b77" +[[package]] +name = "findshlibs" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64" +dependencies = [ + "cc", + "lazy_static", + "libc", + "winapi", +] + [[package]] name = "fixed-hash" version = "0.8.0" @@ -2638,7 +2724,7 @@ dependencies = [ "async-graphql", "async-trait", "axum", - "clap", + "clap 4.3.24", "derive_more", "enum-iterator", "fuel-core-chain-config", @@ -2689,7 +2775,7 @@ version = "0.0.0" dependencies = [ "anyhow", "async-trait", - "clap", + "clap 4.3.24", "criterion", "ctrlc", "ed25519-dalek 1.0.1", @@ -2723,13 +2809,15 @@ name = "fuel-core-bin" version = "0.20.4" dependencies = [ "anyhow", - "clap", + "clap 4.3.24", "const_format", "dirs", "dotenvy", "fuel-core", "humantime", "lazy_static", + "pyroscope", + "pyroscope_pprofrs", "serde_json", "test-case", "tikv-jemallocator", @@ -2785,7 +2873,7 @@ dependencies = [ name = "fuel-core-client-bin" version = "0.20.4" dependencies = [ - "clap", + "clap 4.3.24", "fuel-core-client", "fuel-core-types", "serde_json", @@ -2871,7 +2959,7 @@ name = "fuel-core-keygen" version = "0.20.4" dependencies = [ "anyhow", - "clap", + "clap 4.3.24", "fuel-core-types", "libp2p-core 0.38.0", "serde_json", @@ -3560,6 +3648,15 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hermit-abi" version = "0.3.2" @@ -3957,7 +4054,7 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.2", "libc", "windows-sys 0.48.0", ] @@ -3992,7 +4089,7 @@ version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.2", "rustix 0.38.8", "windows-sys 0.48.0", ] @@ -4030,6 +4127,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd" + [[package]] name = "k256" version = "0.11.6" @@ -4084,6 +4187,26 @@ version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +[[package]] +name = "libflate" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ff4ae71b685bbad2f2f391fe74f6b7659a34871c08b210fdc039e43bee07d18" +dependencies = [ + "adler32", + "crc32fast", + "libflate_lz77", +] + +[[package]] +name = "libflate_lz77" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a52d3a8bfc85f250440e4424db7d857e241a3aebbbe301f3eb606ab15c39acbf" +dependencies = [ + "rle-decode-fast", +] + [[package]] name = "libloading" version = "0.7.4" @@ -4647,7 +4770,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d8de370f98a6cb8a4606618e53e802f93b094ddec0f96988eaec2c27e6e9ce7" dependencies = [ - "clap", + "clap 4.3.24", "termcolor", "threadpool", ] @@ -4767,6 +4890,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.6.5" @@ -4970,6 +5102,16 @@ dependencies = [ "unsigned-varint", ] +[[package]] +name = "names" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bddcd3bf5144b6392de80e04c347cd7fab2508f6df16a85fc496ecd5cec39bc" +dependencies = [ + "clap 3.2.25", + "rand 0.8.5", +] + [[package]] name = "netlink-packet-core" version = "0.4.2" @@ -5129,7 +5271,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.2", "libc", ] @@ -5215,6 +5357,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "os_str_bytes" +version = "6.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d5d9eb14b174ee9aa2ef96dc2b94637a2d4b6e7cb873c7e171f0c20c6cf3eac" + [[package]] name = "overload" version = "0.1.1" @@ -5594,6 +5742,26 @@ dependencies = [ "serde", ] +[[package]] +name = "pprof" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978385d59daf9269189d052ca8a84c1acfd0715c0599a5d5188d4acc078ca46a" +dependencies = [ + "backtrace", + "cfg-if", + "findshlibs", + "libc", + "log", + "nix 0.26.2", + "once_cell", + "parking_lot 0.12.1", + "smallvec", + "symbolic-demangle", + "tempfile", + "thiserror", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -5877,6 +6045,36 @@ dependencies = [ "psl-types", ] +[[package]] +name = "pyroscope" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac8a53ce01af1087eaeee6ce7c4fbf50ea4040ab1825c0115c4bafa039644ba9" +dependencies = [ + "json", + "libc", + "libflate", + "log", + "names", + "prost", + "reqwest", + "thiserror", + "url", + "winapi", +] + +[[package]] +name = "pyroscope_pprofrs" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f010b2a981a7f8449a650f25f309e520b5206ea2d89512dcb146aaa5518ff4" +dependencies = [ + "log", + "pprof", + "pyroscope", + "thiserror", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -6251,6 +6449,12 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "rle-decode-fast" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" + [[package]] name = "rlp" version = "0.5.2" @@ -7175,6 +7379,29 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +[[package]] +name = "symbolic-common" +version = "12.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167a4ffd7c35c143fd1030aa3c2caf76ba42220bd5a6b5f4781896434723b8c3" +dependencies = [ + "debugid", + "memmap2", + "stable_deref_trait", + "uuid 1.4.1", +] + +[[package]] +name = "symbolic-demangle" +version = "12.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e378c50e80686c1c5c205674e1f86a2858bec3d2a7dfdd690331a8a19330f293" +dependencies = [ + "cpp_demangle", + "rustc-demangle", + "symbolic-common", +] + [[package]] name = "syn" version = "1.0.109" @@ -7313,6 +7540,12 @@ dependencies = [ "syn 2.0.29", ] +[[package]] +name = "textwrap" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" + [[package]] name = "thiserror" version = "1.0.47" @@ -8712,7 +8945,7 @@ dependencies = [ name = "xtask" version = "0.0.0" dependencies = [ - "clap", + "clap 4.3.24", "fuel-core", ] diff --git a/bin/fuel-core/src/cli/run.rs b/bin/fuel-core/src/cli/run.rs index df7263ab7f9..0412b6b3af7 100644 --- a/bin/fuel-core/src/cli/run.rs +++ b/bin/fuel-core/src/cli/run.rs @@ -357,8 +357,6 @@ pub async fn exec(command: Command) -> anyhow::Result<()> { "default_network" } .to_string(); - "default_network".to_string() - }; #[cfg(feature = "profiling")] let profiling = command.profiling.clone(); From b00db58ace42a60e0d04f4b8879833fafcc6bbe7 Mon Sep 17 00:00:00 2001 From: Voxelot Date: Fri, 25 Aug 2023 11:50:26 -0700 Subject: [PATCH 3/6] CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2cf97220f0..15fb8660f10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Description of the upcoming release here. ### Added +- [#1324](https://github.com/FuelLabs/fuel-core/pull/1324): Added pyroscope profiling to fuel-core which is enabled via a secondary docker image that has debug symbols enabled. - [#1274](https://github.com/FuelLabs/fuel-core/pull/1274): Added tests to benchmark block synchronization. - [#1309](https://github.com/FuelLabs/fuel-core/pull/1309): Add documentation for running debug builds with CLion and Visual Studio Code. - [#1308](https://github.com/FuelLabs/fuel-core/pull/1308): Add support for loading .env files when compiling with the `env` feature. This allows users to conveniently supply CLI arguments in a secure and IDE-agnostic way. From 83e0ae4532cec12e12c62c6946507b6160681b41 Mon Sep 17 00:00:00 2001 From: Voxelot Date: Mon, 28 Aug 2023 18:58:31 -0700 Subject: [PATCH 4/6] move agent startup to separate fn --- bin/fuel-core/src/cli/run.rs | 47 +++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/bin/fuel-core/src/cli/run.rs b/bin/fuel-core/src/cli/run.rs index 0412b6b3af7..e23774b9db3 100644 --- a/bin/fuel-core/src/cli/run.rs +++ b/bin/fuel-core/src/cli/run.rs @@ -364,25 +364,7 @@ pub async fn exec(command: Command) -> anyhow::Result<()> { // start profiling agent if url is configured #[cfg(feature = "profiling")] - let _profiling_agent = profiling - .pyroscope_url - .as_ref() - .map(|url| -> anyhow::Result<_> { - // Configure profiling backend - let agent = PyroscopeAgent::builder(url, &"fuel-core".to_string()) - .tags(vec![ - ("service", config.name.as_str()), - ("network", network_name.as_str()), - ]) - .backend(pprof_backend( - PprofConfig::new().sample_rate(profiling.pprof_sample_rate), - )) - .build() - .context("failed to start profiler")?; - let agent_running = agent.start().unwrap(); - Ok(agent_running) - }) - .transpose()?; + let _profiling_agent = start_pyroscope_agent(profiling, &config, network_name)?; // log fuel-core version info!("Fuel Core version v{}", env!("CARGO_PKG_VERSION")); @@ -422,6 +404,33 @@ fn load_consensus_key( } } +#[cfg(feature = "profiling")] +fn start_pyroscope_agent( + profiling_args: profiling::ProfilingArgs, + config: &Config, + network_name: String, +) -> anyhow::Result { + profiling_args + .pyroscope_url + .as_ref() + .map(|url| -> anyhow::Result<_> { + // Configure profiling backend + let agent = PyroscopeAgent::builder(url, &"fuel-core".to_string()) + .tags(vec![ + ("service", config.name.as_str()), + ("network", network_name.as_str()), + ]) + .backend(pprof_backend( + PprofConfig::new().sample_rate(profiling_args.pprof_sample_rate), + )) + .build() + .context("failed to start profiler")?; + let agent_running = agent.start().unwrap(); + Ok(agent_running) + }) + .transpose() +} + async fn shutdown_signal() -> anyhow::Result<()> { #[cfg(unix)] { From f62d57a92bcaf8633ba7b6fca1157966cc64f6df Mon Sep 17 00:00:00 2001 From: Voxelot Date: Thu, 31 Aug 2023 16:26:39 -0700 Subject: [PATCH 5/6] remove feature flag for pprof & pyroscope --- .github/workflows/ci.yml | 6 ++---- bin/fuel-core/Cargo.toml | 5 ++--- bin/fuel-core/src/cli/run.rs | 14 ++++++-------- bin/fuel-core/src/cli/run/profiling.rs | 4 ++++ 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b0a002349f..19553087cc4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -305,7 +305,7 @@ jobs: uses: docker/metadata-action@v3 with: images: | - ghcr.io/fuellabs/fuel-core-profiling + ghcr.io/fuellabs/fuel-core-debug tags: | type=sha type=ref,event=branch @@ -329,9 +329,7 @@ jobs: with: context: . file: deployment/Dockerfile - build-args: | - "FEATURES=production,profiling" - "DEBUG_SYMBOLS=true" + build-args: "DEBUG_SYMBOLS=true" push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/bin/fuel-core/Cargo.toml b/bin/fuel-core/Cargo.toml index a0ab5ec043c..1269ff44131 100644 --- a/bin/fuel-core/Cargo.toml +++ b/bin/fuel-core/Cargo.toml @@ -24,8 +24,8 @@ dotenvy = { version = "0.15", optional = true } fuel-core = { workspace = true } humantime = "2.1" lazy_static = { workspace = true } -pyroscope = { version = "0.5", optional = true } -pyroscope_pprofrs = { version = "0.2", optional = true } +pyroscope = "0.5" +pyroscope_pprofrs = "0.2" serde_json = { workspace = true, features = ["raw_value"], optional = true } tikv-jemallocator = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } @@ -49,6 +49,5 @@ p2p = ["fuel-core/p2p", "const_format"] relayer = ["fuel-core/relayer", "dep:url", "dep:serde_json"] rocksdb = ["fuel-core/rocksdb"] rocksdb-production = ["fuel-core/rocksdb-production"] -profiling = ["dep:pyroscope", "dep:pyroscope_pprofrs"] # features to enable in production, but increase build times production = ["env", "metrics", "relayer", "rocksdb-production", "p2p"] diff --git a/bin/fuel-core/src/cli/run.rs b/bin/fuel-core/src/cli/run.rs index e23774b9db3..beb42149dfe 100644 --- a/bin/fuel-core/src/cli/run.rs +++ b/bin/fuel-core/src/cli/run.rs @@ -36,9 +36,10 @@ use fuel_core::{ }, }, }; -#[cfg(feature = "profiling")] -use pyroscope::PyroscopeAgent; -#[cfg(feature = "profiling")] +use pyroscope::{ + pyroscope::PyroscopeAgentRunning, + PyroscopeAgent, +}; use pyroscope_pprofrs::{ pprof_backend, PprofConfig, @@ -358,12 +359,10 @@ pub async fn exec(command: Command) -> anyhow::Result<()> { } .to_string(); - #[cfg(feature = "profiling")] let profiling = command.profiling.clone(); let config = command.get_config()?; // start profiling agent if url is configured - #[cfg(feature = "profiling")] let _profiling_agent = start_pyroscope_agent(profiling, &config, network_name)?; // log fuel-core version @@ -404,12 +403,11 @@ fn load_consensus_key( } } -#[cfg(feature = "profiling")] -fn start_pyroscope_agent( +fn start_pyroscope_agent( profiling_args: profiling::ProfilingArgs, config: &Config, network_name: String, -) -> anyhow::Result { +) -> anyhow::Result>> { profiling_args .pyroscope_url .as_ref() diff --git a/bin/fuel-core/src/cli/run/profiling.rs b/bin/fuel-core/src/cli/run/profiling.rs index 2471e02d66e..fda403990d5 100644 --- a/bin/fuel-core/src/cli/run/profiling.rs +++ b/bin/fuel-core/src/cli/run/profiling.rs @@ -2,9 +2,13 @@ use clap::Args; #[derive(Debug, Clone, Args)] pub struct ProfilingArgs { + /// Enables realtime profiling with pyroscope if set, and streams results to the pyroscope endpoint. + /// For best results, the binary should be built with debug symbols included. #[clap(long = "pyroscope-url", env)] pub pyroscope_url: Option, + /// Pyroscope sample frequency in hertz. A higher sample rate improves profiling granularity + /// at the cost of additional measurement overhead. #[clap(long = "pprof-sample-rate", default_value = "100", env)] pub pprof_sample_rate: u32, } From 77caf74b426cf39f173d37495e1c07c58f6c2a43 Mon Sep 17 00:00:00 2001 From: Voxelot Date: Thu, 31 Aug 2023 17:05:02 -0700 Subject: [PATCH 6/6] reword changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd6c02c759e..9bb14791f5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ Description of the upcoming release here. ### Added -- [#1324](https://github.com/FuelLabs/fuel-core/pull/1324): Added pyroscope profiling to fuel-core which is enabled via a secondary docker image that has debug symbols enabled. +- [#1324](https://github.com/FuelLabs/fuel-core/pull/1324): Added pyroscope profiling to fuel-core, intended to be used by a secondary docker image that has debug symbols enabled. - [#1309](https://github.com/FuelLabs/fuel-core/pull/1309): Add documentation for running debug builds with CLion and Visual Studio Code. - [#1308](https://github.com/FuelLabs/fuel-core/pull/1308): Add support for loading .env files when compiling with the `env` feature. This allows users to conveniently supply CLI arguments in a secure and IDE-agnostic way. - [#1304](https://github.com/FuelLabs/fuel-core/pull/1304): Implemented `submit_and_await_commit_with_receipts` method for `FuelClient`.