Timeline support (#9533)

Co-authored-by: Олег <150132506+iddqdex@users.noreply.github.com>
ydb-platform · Sep 20, 2024 · c006f27 · c006f27
1 parent d40fa72
commit c006f27
Show file tree

Hide file tree

Showing 19 changed files with 1,995 additions and 16 deletions.
diff --git a/ydb/core/fq/libs/compute/common/utils.cpp b/ydb/core/fq/libs/compute/common/utils.cpp
@@ -1,4 +1,5 @@
 #include "utils.h"
+#include <ydb/public/lib/ydb_cli/common/plan2svg.h>
 
 #include <library/cpp/json/json_reader.h>
 #include <library/cpp/json/json_writer.h>
@@ -430,7 +431,7 @@ void EnumeratePlans(NYson::TYsonWriter& writer, NJson::TJsonValue& value, ui32&
     }
 }
 
-TString GetV1StatFromV2Plan(const TString& plan, double* cpuUsage) {
+TString GetV1StatFromV2Plan(const TString& plan, double* cpuUsage, TString* timeline, ui64 maxTimelineSize) {
     TStringStream out;
     NYson::TYsonWriter writer(&out);
     writer.OnBeginMap();
@@ -471,6 +472,24 @@ TString GetV1StatFromV2Plan(const TString& plan, double* cpuUsage) {
             }
         }
     }
+    if (timeline) {
+        TPlanVisualizer planViz;
+        planViz.LoadPlans(plan);
+        *timeline = planViz.PrintSvgSafe();
+        if (maxTimelineSize && timeline->size() > maxTimelineSize) {
+            TStringBuilder builder;
+            builder
+                << "<svg width='600' height='200' xmlns='http://www.w3.org/2000/svg'>" << Endl
+                << "  <text font-size='16px' x='20' y='40'>There is nothing wrong with the request.</text>" << Endl
+                << "  <text font-size='16px' x='20' y='80'>Unfortunately, image size " << timeline->size() << " is too large.</text>" << Endl
+                << "  <text font-size='16px' x='20' y='120'>It exceeds limit of " << maxTimelineSize << " and was discarded</text>" << Endl
+                << "</svg>" << Endl;
+            *timeline = builder;
+        }
+        // remove json "timeline" field after migration
+        writer.OnKeyedItem("timeline");
+        writer.OnStringScalar(*timeline);
+    }
     writer.OnEndMap();
     return NJson2Yson::ConvertYson2Json(out.Str());
 }
@@ -1145,7 +1164,7 @@ struct TNoneStatProcessor : IPlanStatProcessor {
         return plan;
     }
 
-    TString GetQueryStat(const TString&, double& cpuUsage) override {
+    TString GetQueryStat(const TString&, double& cpuUsage, TString*, ui64) override {
         cpuUsage = 0.0;
         return "";
     }
@@ -1178,8 +1197,8 @@ struct TPlanStatProcessor : IPlanStatProcessor {
         return plan;
     }
 
-    TString GetQueryStat(const TString& plan, double& cpuUsage) override {
-        return GetV1StatFromV2Plan(plan, &cpuUsage);
+    TString GetQueryStat(const TString& plan, double& cpuUsage, TString* timeline, ui64 maxtimelineSize) override {
+        return GetV1StatFromV2Plan(plan, &cpuUsage, timeline, maxtimelineSize);
     }
 
     TPublicStat GetPublicStat(const TString& stat) override {
@@ -1210,8 +1229,8 @@ struct TProfileStatProcessor : TPlanStatProcessor {
 };
 
 struct TProdStatProcessor : TFullStatProcessor {
-    TString GetQueryStat(const TString& plan, double& cpuUsage) override {
-        return GetPrettyStatistics(GetV1StatFromV2Plan(plan, &cpuUsage));
+    TString GetQueryStat(const TString& plan, double& cpuUsage, TString* timeline, ui64 maxtimelineSize) override {
+        return GetPrettyStatistics(GetV1StatFromV2Plan(plan, &cpuUsage, timeline, maxtimelineSize));
     }
 };
 
@@ -1229,8 +1248,12 @@ std::unique_ptr<IPlanStatProcessor> CreateStatProcessor(const TString& statViewN
 
 PingTaskRequestBuilder::PingTaskRequestBuilder(const NConfig::TCommonConfig& commonConfig, std::unique_ptr<IPlanStatProcessor>&& processor) 
     : Compressor(commonConfig.GetQueryArtifactsCompressionMethod(), commonConfig.GetQueryArtifactsCompressionMinSize())
-    , Processor(std::move(processor))
-{}
+    , Processor(std::move(processor)), ShowQueryTimeline(commonConfig.GetShowQueryTimeline()), MaxQueryTimelineSize(commonConfig.GetMaxQueryTimelineSize())
+{
+    if (!MaxQueryTimelineSize) {
+        MaxQueryTimelineSize = 200_KB;
+    }
+}
 
 Fq::Private::PingTaskRequest PingTaskRequestBuilder::Build(
     const Ydb::TableStats::QueryStats& queryStats, 
@@ -1294,9 +1317,13 @@ Fq::Private::PingTaskRequest PingTaskRequestBuilder::Build(const TString& queryP
 
     CpuUsage = 0.0;
     try {
-        auto stat = Processor->GetQueryStat(plan, CpuUsage);
+        TString timeline;
+        auto stat = Processor->GetQueryStat(plan, CpuUsage, ShowQueryTimeline ? &timeline : nullptr, MaxQueryTimelineSize);
         pingTaskRequest.set_statistics(stat);
         pingTaskRequest.set_dump_raw_statistics(true);
+        if (timeline) {
+            pingTaskRequest.set_timeline(timeline);
+        }
         auto flatStat = Processor->GetFlatStat(plan);
         flatStat["CompilationTimeUs"] = compilationTimeUs;
         flatStat["ComputeTimeUs"] = computeTimeUs;

diff --git a/ydb/core/fq/libs/compute/common/utils.h b/ydb/core/fq/libs/compute/common/utils.h
@@ -28,7 +28,7 @@ inline std::shared_ptr<NYdb::NTable::TTableClient> CreateNewTableClient(const TS
                                                         tableSettings);
 }
 
-TString GetV1StatFromV2Plan(const TString& plan, double* cpuUsage = nullptr);
+TString GetV1StatFromV2Plan(const TString& plan, double* cpuUsage = nullptr, TString* timeline = nullptr, ui64 maxTimelineSize = 0);
 TString GetV1StatFromV2PlanV2(const TString& plan);
 TString GetPrettyStatistics(const TString& statistics);
 THashMap<TString, i64> AggregateStats(TStringBuf plan);
@@ -55,7 +55,7 @@ struct IPlanStatProcessor {
     virtual Ydb::Query::StatsMode GetStatsMode() = 0;
     virtual TString ConvertPlan(const TString& plan) = 0;
     virtual TString GetPlanVisualization(const TString& plan) = 0;
-    virtual TString GetQueryStat(const TString& plan, double& cpuUsage) = 0;
+    virtual TString GetQueryStat(const TString& plan, double& cpuUsage, TString* timeline, ui64 maxtimelineSize) = 0;
     virtual TPublicStat GetPublicStat(const TString& stat) = 0;
     virtual THashMap<TString, i64> GetFlatStat(TStringBuf plan) = 0;
 };
@@ -79,6 +79,8 @@ class PingTaskRequestBuilder {
 private:
     const TCompressor Compressor;
     std::unique_ptr<IPlanStatProcessor> Processor;
+    bool ShowQueryTimeline = false;
+    ui64 MaxQueryTimelineSize = 0;
 };
 
 TString GetStatViewName(const ::NFq::TRunActorParams& params);

diff --git a/ydb/core/fq/libs/compute/common/ya.make b/ydb/core/fq/libs/compute/common/ya.make
@@ -19,6 +19,7 @@ PEERDIR(
     ydb/library/yql/providers/generic/connector/api/service/protos
     ydb/library/yql/providers/generic/connector/libcpp
     ydb/library/yql/providers/s3/actors_factory
+    ydb/public/lib/ydb_cli/common
 )
 
 YQL_LAST_ABI_VERSION()

diff --git a/ydb/core/fq/libs/config/protos/common.proto b/ydb/core/fq/libs/config/protos/common.proto
@@ -29,4 +29,6 @@ message TCommonConfig {
     bool KeepInternalErrors = 13;
     bool UseNativeProtocolForClickHouse = 14;
     bool DisableSslForGenericDataSources = 15;
+    bool ShowQueryTimeline = 16;
+    uint64 MaxQueryTimelineSize = 17; // default: 200KB
 }
diff --git a/ydb/core/fq/libs/control_plane_storage/internal/task_ping.cpp b/ydb/core/fq/libs/control_plane_storage/internal/task_ping.cpp
@@ -285,6 +285,10 @@ TPingTaskParams ConstructHardPingTask(
             internal.set_current_load(request.current_load());
         }
 
+        if (request.timeline()) {
+            internal.set_timeline(request.timeline());
+        }
+
         if (request.flat_stats_size() != 0) {
             internal.clear_statistics();
             auto stats = DeserializeFlatStats(request.flat_stats());

diff --git a/ydb/core/fq/libs/control_plane_storage/proto/yq_internal.proto b/ydb/core/fq/libs/control_plane_storage/proto/yq_internal.proto
@@ -57,6 +57,7 @@ message QueryInternal {
     NYql.NDqProto.StatusIds.StatusCode pending_status_code = 28;
     repeated StatisticsNamedValue statistics = 29;
     int32 current_load = 30;
+    string timeline = 31;
 }
 
 message JobInternal {

diff --git a/ydb/core/fq/libs/control_plane_storage/util.cpp b/ydb/core/fq/libs/control_plane_storage/util.cpp
@@ -179,6 +179,7 @@ bool DoesPingTaskUpdateQueriesTable(const Fq::Private::PingTaskRequest& request)
         || !request.issues().empty()
         || !request.transient_issues().empty()
         || request.statistics()
+        || request.timeline()
         || !request.result_set_meta().empty()
         || request.ast()
         || request.ast_compressed().data()

diff --git a/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp b/ydb/core/fq/libs/control_plane_storage/ydb_control_plane_storage_queries.cpp
@@ -594,6 +594,12 @@ void TYdbControlPlaneStorageActor::Handle(TEvControlPlaneStorage::TEvDescribeQue
                 }
             }
         }
+
+        auto timeline = internal.timeline();
+        if (timeline) {
+            result.mutable_query()->mutable_timeline()->set_svg(timeline);
+        }
+
         if (!permissions.Check(TPermissions::VIEW_AST)) {
             result.mutable_query()->clear_ast();
         } else {

diff --git a/ydb/core/fq/libs/protos/fq_private.proto b/ydb/core/fq/libs/protos/fq_private.proto
@@ -166,6 +166,7 @@ message PingTaskRequest {
     bool dump_raw_statistics = 38;
     repeated Ydb.ValuePair flat_stats = 39;
     int32 current_load = 40;
+    string timeline = 41;
 }
 
 message PingTaskResult {

diff --git a/ydb/public/api/protos/draft/fq.proto b/ydb/public/api/protos/draft/fq.proto
@@ -205,6 +205,10 @@ message ResultSetMeta {
     bool truncated = 3;
 }
 
+message QueryTimeline {
+    string svg = 1; // No validation because generated on server side
+}
+
 message Query {
     QueryMeta meta = 1;
     QueryContent content = 2;
@@ -214,6 +218,7 @@ message Query {
     QueryStatistics statistics = 6;
     repeated ResultSetMeta result_set_meta = 7;
     QueryAst ast = 8;
+    QueryTimeline timeline = 9;
 }
 
 message QueryStatistics {

diff --git a/ydb/public/lib/ydb_cli/commands/ydb_benchmark.cpp b/ydb/public/lib/ydb_cli/commands/ydb_benchmark.cpp
@@ -1,7 +1,8 @@
 #include "ydb_benchmark.h"
 #include "benchmark_utils.h"
-#include <ydb/public/lib/ydb_cli/common/pretty_table.h>
 #include <ydb/public/lib/ydb_cli/common/format.h>
+#include <ydb/public/lib/ydb_cli/common/plan2svg.h>
+#include <ydb/public/lib/ydb_cli/common/pretty_table.h>
 #include <library/cpp/json/json_writer.h>
 #include <util/string/printf.h>
 #include <util/folder/path.h>
@@ -93,7 +94,7 @@ TString TWorkloadCommandBenchmark::PatchQuery(const TStringBuf& original) const
 
     std::vector<TStringBuf> lines;
     for (auto& line : StringSplitter(result).Split('\n').SkipEmpty()) {
-        if (line.StartsWith("--")) {
+        if (line.StartsWith("--") && !line.StartsWith("--!")) {
             continue;
         }
 
@@ -386,6 +387,12 @@ bool TWorkloadCommandBenchmark::RunBench(TClient& client, NYdbWorkload::IWorkloa
                     TFileOutput out(PlanFileName + ".ast");
                     out << res.GetPlanAst();
                 }
+                {
+                    TPlanVisualizer pv;
+                    pv.LoadPlans(res.GetQueryPlan());
+                    TFileOutput out(PlanFileName + ".svg");
+                    out << pv.PrintSvgSafe();
+                }
                 planSaved = true;
             }
         }