[opt](cpu-profile) enable cpu profile in BE webui (#40330)

.licenserc.yaml

@@ -93,4 +93,5 @@ header:
     - "pytest/sys/data"
     - "pytest/deploy/*.conf"
     - "tools/jeprof"
+    - "tools/FlameGraph/*"
   comment: on-failure

LICENSE.txt

@@ -725,4 +725,12 @@ Apache 2.0, Copyright 2023 SAP SE or an SAP affiliate company, Johannes Bechberg
 
 This project is maintained by the SapMachine team at SAP SE
 
-----------------------------------------------------------------------------------
+----------------------------------------------------------------------------------
+
+be/tools/FlameGraph/*.pl: COMMON DEVELOPMENT AND DISTRIBUTION LICENSE Version 1.0
+
+Unless otherwise noted, all files in this distribution are released
+under the Common Development and Distribution License (CDDL).
+Exceptions are noted within the associated source files.
+
+----------------------------------------------------------------------------------

be/src/http/action/pprof_actions.cpp

@@ -17,9 +17,15 @@
 
 #include "http/action/pprof_actions.h"
 
+#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \
+        !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC)
 #include <gperftools/heap-profiler.h>    // IWYU pragma: keep
 #include <gperftools/malloc_extension.h> // IWYU pragma: keep
-#include <gperftools/profiler.h>         // IWYU pragma: keep
+#endif
+#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \
+        !defined(THREAD_SANITIZER)
+#include <gperftools/profiler.h> // IWYU pragma: keep
+#endif
 #include <stdio.h>
 
 #include <fstream>
@@ -133,8 +139,7 @@ class ProfileAction : public HttpHandler {
 };
 
 void ProfileAction::handle(HttpRequest* req) {
-#if defined(ADDRESS_SANITIZER) || defined(LEAK_SANITIZER) || defined(THREAD_SANITIZER) || \
-        defined(USE_JEMALLOC)
+#if defined(ADDRESS_SANITIZER) || defined(LEAK_SANITIZER) || defined(THREAD_SANITIZER)
     std::string str = "CPU profiling is not available with address sanitizer or jemalloc builds.";
     HttpChannel::send_reply(req, str);
 #else
@@ -170,6 +175,7 @@ void ProfileAction::handle(HttpRequest* req) {
             prof_file.close();
             std::string str = ss.str();
             HttpChannel::send_reply(req, str);
+            return;
         }
 
         // text type. we will return readable content via http response
@@ -185,7 +191,7 @@ void ProfileAction::handle(HttpRequest* req) {
         std::string svg_file_content;
         std::string flamegraph_install_dir =
                 std::string(std::getenv("DORIS_HOME")) + "/tools/FlameGraph/";
-        Status st = PprofUtils::generate_flamegraph(30, flamegraph_install_dir, false,
+        Status st = PprofUtils::generate_flamegraph(seconds, flamegraph_install_dir, false,
                                                     &svg_file_content);
         if (!st.ok()) {
             HttpChannel::send_reply(req, st.to_string());

be/src/http/default_path_handlers.cpp

@@ -281,8 +281,7 @@ void heap_handler(const WebPageHandler::ArgumentMap& args, std::stringstream* ou
 void cpu_handler(const WebPageHandler::ArgumentMap& args, std::stringstream* output) {
     (*output) << "<h2>CPU Profile</h2>" << std::endl;
 
-#if defined(ADDRESS_SANITIZER) || defined(LEAK_SANITIZER) || defined(THREAD_SANITIZER) || \
-        defined(USE_JEMALLOC)
+#if defined(ADDRESS_SANITIZER) || defined(LEAK_SANITIZER) || defined(THREAD_SANITIZER)
     (*output) << "<pre>" << std::endl;
     (*output) << "CPU profiling is not available with address sanitizer builds." << std::endl;
     (*output) << "</pre>" << std::endl;
@@ -313,7 +312,8 @@ void cpu_handler(const WebPageHandler::ArgumentMap& args, std::stringstream* out
               << std::endl;
     (*output) << "And you need to download the FlameGraph and place it under 'be/tools/FlameGraph'."
               << std::endl;
-    (*output) << "Finally, check if the following files exist" << std::endl;
+    (*output) << "Finally, check if the following files exist. And should be executable."
+              << std::endl;
     (*output) << std::endl;
     (*output) << "    be/tools/FlameGraph/stackcollapse-perf.pl" << std::endl;
     (*output) << "    be/tools/FlameGraph/flamegraph.pl" << std::endl;
@@ -333,9 +333,6 @@ void cpu_handler(const WebPageHandler::ArgumentMap& args, std::stringstream* out
             << std::endl;
     (*output) << "    <br/>" << std::endl;
     (*output) << "    <div id=\"cpuResult\"><pre id=\"cpuContent\"></pre></div>" << std::endl;
-    (*output) << "    <br/>" << std::endl;
-    (*output) << "    <div id=\"cpuResultGraph\"><pre id=\"cpuContentGraph\"></pre></div>"
-              << std::endl;
     (*output) << "</div>" << std::endl;
 
     // for text profile
@@ -348,36 +345,36 @@ void cpu_handler(const WebPageHandler::ArgumentMap& args, std::stringstream* out
     (*output) << "        type: \"GET\"," << std::endl;
     (*output) << "        dataType: \"text\"," << std::endl;
     (*output) << "        url: \"pprof/profile?type=text\"," << std::endl;
-    (*output) << "        timeout: 60000," << std::endl;
+    (*output) << "        timeout: 120000," << std::endl;
     (*output) << "        success: function (result) {" << std::endl;
     (*output) << "            document.getElementById(\"cpuContent\").innerText = result;"
               << std::endl;
     (*output) << "        }" << std::endl;
     (*output) << "        ," << std::endl;
     (*output) << "        error: function (result) {" << std::endl;
-    (*output) << "            alert(result);" << std::endl;
+    (*output) << "            alert(JSON.stringify(result));" << std::endl;
     (*output) << "        }" << std::endl;
     (*output) << "        ," << std::endl;
     (*output) << "    });" << std::endl;
     (*output) << "});" << std::endl;
 
     // for graph profile
     (*output) << "$('#getCpuGraph').click(function () {" << std::endl;
-    (*output) << "    document.getElementById(\"cpuContentGraph\").innerText = \"Sampling... (30 "
+    (*output) << "    document.getElementById(\"cpuContent\").innerText = \"Sampling... (30 "
                  "seconds)\";"
               << std::endl;
     (*output) << "    $.ajax({" << std::endl;
     (*output) << "        type: \"GET\"," << std::endl;
     (*output) << "        dataType: \"text\"," << std::endl;
     (*output) << "        url: \"pprof/profile?type=flamegraph\"," << std::endl;
-    (*output) << "        timeout: 60000," << std::endl;
+    (*output) << "        timeout: 120000," << std::endl;
     (*output) << "        success: function (result) {" << std::endl;
-    (*output) << "            document.getElementById(\"cpuResultGraph\").innerHTML = result;"
+    (*output) << "            document.getElementById(\"cpuContent\").innerHTML = result;"
               << std::endl;
     (*output) << "        }" << std::endl;
     (*output) << "        ," << std::endl;
     (*output) << "        error: function (result) {" << std::endl;
-    (*output) << "            alert(result);" << std::endl;
+    (*output) << "            alert(JSON.stringify(result));" << std::endl;
     (*output) << "        }" << std::endl;
     (*output) << "        ," << std::endl;
     (*output) << "    });" << std::endl;

be/src/util/pprof_utils.cpp

@@ -114,6 +114,7 @@ Status PprofUtils::get_readable_profile(const std::string& file_or_content, bool
     std::string final_cmd =
             pprof_cmd + strings::Substitute(" --text $0 $1", self_cmdline, final_file);
     AgentUtils util;
+    LOG(INFO) << "begin to run command: " << final_cmd;
     bool rc = util.exec_cmd(final_cmd, &cmd_output, false);
 
     // delete raw file
@@ -158,6 +159,7 @@ Status PprofUtils::generate_flamegraph(int32_t sample_seconds,
 
     AgentUtils util;
     std::string cmd_output;
+    LOG(INFO) << "begin to run command: " << cmd.str();
     bool rc = util.exec_cmd(cmd.str(), &cmd_output);
     if (!rc) {
         static_cast<void>(io::global_local_filesystem()->delete_file(tmp_file.str()));
@@ -174,6 +176,7 @@ Status PprofUtils::generate_flamegraph(int32_t sample_seconds,
         std::stringstream gen_cmd;
         gen_cmd << perf_cmd << " script -i " << tmp_file.str() << " | " << stackcollapse_perf_pl
                 << " | " << flamegraph_pl << " > " << graph_file.str();
+        LOG(INFO) << "begin to run command: " << gen_cmd.str();
         rc = util.exec_cmd(gen_cmd.str(), &res_content);
         if (!rc) {
             static_cast<void>(io::global_local_filesystem()->delete_file(tmp_file.str()));
@@ -185,6 +188,7 @@ Status PprofUtils::generate_flamegraph(int32_t sample_seconds,
         std::stringstream gen_cmd;
         gen_cmd << perf_cmd << " script -i " << tmp_file.str() << " | " << stackcollapse_perf_pl
                 << " | " << flamegraph_pl;
+        LOG(INFO) << "begin to run command: " << gen_cmd.str();
         rc = util.exec_cmd(gen_cmd.str(), &res_content, false);
         if (!rc) {
             static_cast<void>(io::global_local_filesystem()->delete_file(tmp_file.str()));

be/src/vec/exec/scan/vfile_scanner.cpp

@@ -826,7 +826,7 @@ Status VFileScanner::_get_next_reader() {
             std::unique_ptr<ParquetReader> parquet_reader = ParquetReader::create_unique(
                     _profile, *_params, range, _state->query_options().batch_size,
                     const_cast<cctz::time_zone*>(&_state->timezone_obj()), _io_ctx.get(), _state,
-                    _shoudl_enable_file_meta_cache() ? ExecEnv::GetInstance()->file_meta_cache()
+                    _should_enable_file_meta_cache() ? ExecEnv::GetInstance()->file_meta_cache()
                                                      : nullptr,
                     _state->query_options().enable_parquet_lazy_mat);
             {

be/src/vec/exec/scan/vfile_scanner.h

@@ -243,7 +243,7 @@ class VFileScanner : public VScanner {
     // 1. max_external_file_meta_cache_num is > 0
     // 2. the file number is less than 1/3 of cache's capacibility
     // Otherwise, the cache miss rate will be high
-    bool _shoudl_enable_file_meta_cache() {
+    bool _should_enable_file_meta_cache() {
         return config::max_external_file_meta_cache_num > 0 &&
                _split_source->num_scan_ranges() < config::max_external_file_meta_cache_num / 3;
     }

build.sh

@@ -666,7 +666,8 @@ if [[ "${OUTPUT_BE_BINARY}" -eq 1 ]]; then
     install -d "${DORIS_OUTPUT}/be/bin" \
         "${DORIS_OUTPUT}/be/conf" \
         "${DORIS_OUTPUT}/be/lib" \
-        "${DORIS_OUTPUT}/be/www"
+        "${DORIS_OUTPUT}/be/www" \
+        "${DORIS_OUTPUT}/be/tools/FlameGraph"
 
     cp -r -p "${DORIS_HOME}/be/output/bin"/* "${DORIS_OUTPUT}/be/bin"/
     cp -r -p "${DORIS_HOME}/be/output/conf"/* "${DORIS_OUTPUT}/be/conf"/
@@ -712,6 +713,7 @@ EOF
     fi
 
     cp -r -p "${DORIS_HOME}/webroot/be"/* "${DORIS_OUTPUT}/be/www"/
+    cp -r -p "${DORIS_HOME}/tools/FlameGraph"/* "${DORIS_OUTPUT}/be/tools/FlameGraph"/
     if [[ "${STRIP_DEBUG_INFO}" = "ON" ]]; then
         cp -r -p "${DORIS_HOME}/be/output/lib/debug_info" "${DORIS_OUTPUT}/be/lib"/
     fi

dist/LICENSE-dist.txt

@@ -1503,35 +1503,36 @@ LGPL -- licenes/LICENSE-LGPL.txt
     * gsasl: 1.10.0/1.8.0
 
 Other dependencies:
-    * libevent: 2.1.12 -- license/LICENSE-libevent.txt
-    * openssl: 1.1.1s -- license/LICENSE-openssl.txt
-    * gflag: 2.2.2 -- license/LICENSE-gflag.txt
-    * glog: 0.4.0 -- license/LICENSE-glog.txt
-    * gtest: 1.11.0 -- license/LICENSE-gtest.txt
-    * snappy: 1.1.8 -- license/LICENSE-snappy.txt
-    * gperftools: 2.9.1 -- license/LICENSE-gperftools.txt
-    * zlib: 1.2.11 -- license/LICENSE-zlib.txt
-    * lz4: 1.9.3 -- license/LICENSE-lz4.txt
-    * bzip2: 1.0.8 -- license/LICENSE-bzip2.txt
-    * rapidjson@1a803826 -- license/LICENSE-rapidjson.txt
-    * curl: 7.79.0 -- license/LICENSE-curl.txt
-    * re2: 2021-02-02 -- license/LICENSE-re2.txt
-    * hyperscan: 5.4.0 -- license/LICENSE-hyperscan.txt
-    * vectorscan: 5.4.7 -- license/LICENSE-vectorscan.txt
-    * boost: 1.73.0 -- license/LICENSE-boost.txt
-    * unixodbc: 2.3.7 -- license/LICENSE-unixodbc.txt
-    * leveldb: 1.23 -- license/LICENSE-leveldb.txt
-    * cyrus-sasl: 2.1.27 -- license/LICENSE-cyrus-sasl.txt
-    * librdkafka: 1.8.2 -- license/LICENSE-librdkafka.txt
-    * zstd: 1.5.2 -- license/LICENSE-zstd.txt
-    * brotli: 1.0.9 -- license/LICENSE-brotli.txt
-    * bitshuffle: 0.5.1 -- license/LICENSE-bigshuffle.txt
-    * fmt: 7.1.3 -- license/LICENSE-fmt.txt
-    * jemalloc: 5.3.0 -- license/LICENSE-jemolloc.txt
-    * lzma@master -- license/LICENSE-lzma.txt
-    * libdivide: 5.0 -- license/LICENSE-libdivide.txt
-    * pdqsort: 0.0.0+git20180419 -- license/LICENSE-pdqsort.txt
-    * breakpad@38ee0be -- license/LICENSE-breakpod.txt
-    * xsimd: xmid@e9234cd6  -- license/LICENSE-xsimd.txt
-    * xxhash: 0.8.1  -- license/LICENSE-xxhash.txt
-    * concurrentqueue: 1.0.3  -- license/LICENSE-concurrentqueue.txt
+    * libevent: 2.1.12 -- licenses/LICENSE-libevent.txt
+    * openssl: 1.1.1s -- licenses/LICENSE-openssl.txt
+    * gflag: 2.2.2 -- licenses/LICENSE-gflag.txt
+    * glog: 0.4.0 -- licenses/LICENSE-glog.txt
+    * gtest: 1.11.0 -- licenses/LICENSE-gtest.txt
+    * snappy: 1.1.8 -- licenses/LICENSE-snappy.txt
+    * gperftools: 2.9.1 -- licenses/LICENSE-gperftools.txt
+    * zlib: 1.2.11 -- licenses/LICENSE-zlib.txt
+    * lz4: 1.9.3 -- licenses/LICENSE-lz4.txt
+    * bzip2: 1.0.8 -- licenses/LICENSE-bzip2.txt
+    * rapidjson@1a803826 -- licenses/LICENSE-rapidjson.txt
+    * curl: 7.79.0 -- licenses/LICENSE-curl.txt
+    * re2: 2021-02-02 -- licenses/LICENSE-re2.txt
+    * hyperscan: 5.4.0 -- licenses/LICENSE-hyperscan.txt
+    * vectorscan: 5.4.7 -- licenses/LICENSE-vectorscan.txt
+    * boost: 1.73.0 -- licenses/LICENSE-boost.txt
+    * unixodbc: 2.3.7 -- licenses/LICENSE-unixodbc.txt
+    * leveldb: 1.23 -- licenses/LICENSE-leveldb.txt
+    * cyrus-sasl: 2.1.27 -- licenses/LICENSE-cyrus-sasl.txt
+    * librdkafka: 1.8.2 -- licenses/LICENSE-librdkafka.txt
+    * zstd: 1.5.2 -- licenses/LICENSE-zstd.txt
+    * brotli: 1.0.9 -- licenses/LICENSE-brotli.txt
+    * bitshuffle: 0.5.1 -- licenses/LICENSE-bigshuffle.txt
+    * fmt: 7.1.3 -- licenses/LICENSE-fmt.txt
+    * jemalloc: 5.3.0 -- licenses/LICENSE-jemolloc.txt
+    * lzma@master -- licenses/LICENSE-lzma.txt
+    * libdivide: 5.0 -- licenses/LICENSE-libdivide.txt
+    * pdqsort: 0.0.0+git20180419 -- licenses/LICENSE-pdqsort.txt
+    * breakpad@38ee0be -- licenses/LICENSE-breakpod.txt
+    * xsimd: xmid@e9234cd6  -- licenses/LICENSE-xsimd.txt
+    * xxhash: 0.8.1  -- licenses/LICENSE-xxhash.txt
+    * concurrentqueue: 1.0.3  -- licenses/LICENSE-concurrentqueue.txt
+    * FlameGraph -- licenses/LICENSE-CDDL-1.0.txt

tools/FlameGraph/README

@@ -0,0 +1,13 @@
+These 2 files:
+
+- flamegraph.pl
+- stackcollapse-perf.pl
+
+are copied from:
+
+https://github.com/brendangregg/FlameGraph/blob/master/flamegraph.pl
+https://github.com/brendangregg/FlameGraph/blob/master/stackcollapse-perf.pl
+
+which are under license:
+
+https://github.com/brendangregg/FlameGraph/blob/master/docs/cddl1.txt