From 746b18271f837d6fc0a9dfde787f12373d589bda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9E=D0=BB=D0=B5=D0=B3?= <150132506+iddqdex@users.noreply.github.com> Date: Fri, 14 Jun 2024 09:30:12 +0300 Subject: [PATCH] Pretty view for workload run (#5531) --- .../lib/ydb_cli/commands/benchmark_utils.cpp | 46 ++- .../lib/ydb_cli/commands/benchmark_utils.h | 28 +- .../lib/ydb_cli/commands/ydb_benchmark.cpp | 269 ++++++++++++++---- .../lib/ydb_cli/commands/ydb_benchmark.h | 2 + 4 files changed, 264 insertions(+), 81 deletions(-) diff --git a/ydb/public/lib/ydb_cli/commands/benchmark_utils.cpp b/ydb/public/lib/ydb_cli/commands/benchmark_utils.cpp index da548a660d2b..51e3ada4c799 100644 --- a/ydb/public/lib/ydb_cli/commands/benchmark_utils.cpp +++ b/ydb/public/lib/ydb_cli/commands/benchmark_utils.cpp @@ -70,6 +70,7 @@ TTestInfo::TTestInfo(std::vector&& clientTimings, std::vector(ServerTimings.size()); auto serverTimingsCopy = ServerTimings; + Sort(serverTimingsCopy); auto centerElement = serverTimingsCopy.begin() + ServerTimings.size() / 2; std::nth_element(serverTimingsCopy.begin(), centerElement, serverTimingsCopy.end()); @@ -79,6 +80,36 @@ TTestInfo::TTestInfo(std::vector&& clientTimings, std::vectorMilliSeconds(); } + const auto ubCount = std::max(1, serverTimingsCopy.size() * 2 / 3); + double ub = 1; + for (ui32 i = 0; i < ubCount; ++i) { + ub *= serverTimingsCopy[i].MillisecondsFloat(); + } + UnixBench = TDuration::MilliSeconds(pow(ub, 1. / ubCount)); +} + +void TTestInfo::operator /=(const ui32 count) { + ColdTime /= count; + Min /= count; + Max /= count; + RttMin /= count; + RttMax /= count; + RttMean /= count; + Mean /= count; + Median /= count; + UnixBench /= count; +} + +void TTestInfo::operator +=(const TTestInfo& other) { + ColdTime += other.ColdTime; + Min += other.Min; + Max += other.Max; + RttMin += other.RttMin; + RttMax += other.RttMax; + RttMean += other.RttMean; + Mean += other.Mean; + Median += other.Median; + UnixBench += other.UnixBench; } TString FullTablePath(const TString& database, const TString& table) { @@ -106,8 +137,9 @@ bool HasCharsInString(const TString& str) { class IQueryResultScanner { private: - TString ErrorInfo; - TDuration ServerTiming; + YDB_READONLY_DEF(TString, ErrorInfo); + YDB_READONLY_DEF(TDuration, ServerTiming); + public: virtual ~IQueryResultScanner() = default; virtual void OnStart(const TVector& columns) = 0; @@ -118,12 +150,6 @@ class IQueryResultScanner { void OnError(const TString& info) { ErrorInfo = info; } - const TString& GetErrorInfo() const { - return ErrorInfo; - } - TDuration GetServerTiming() const { - return ServerTiming; - } template bool Scan(TIterator& it) { @@ -139,12 +165,12 @@ class IQueryResultScanner { if constexpr (std::is_same_v) { if (streamPart.HasQueryStats()) { - ServerTiming = streamPart.GetQueryStats().GetTotalDuration(); + ServerTiming += streamPart.GetQueryStats().GetTotalDuration(); } } else { const auto& stats = streamPart.GetStats(); if (stats) { - ServerTiming = stats->GetTotalDuration(); + ServerTiming += stats->GetTotalDuration(); } } diff --git a/ydb/public/lib/ydb_cli/commands/benchmark_utils.h b/ydb/public/lib/ydb_cli/commands/benchmark_utils.h index d4d3f9a6ae0d..0e781659e8a6 100644 --- a/ydb/public/lib/ydb_cli/commands/benchmark_utils.h +++ b/ydb/public/lib/ydb_cli/commands/benchmark_utils.h @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -20,10 +21,13 @@ struct TTestInfo { double Mean = 0; double Median = 0; double Std = 0; + TDuration UnixBench; std::vector ClientTimings; // timings captured by the client application. these timings include time RTT between server and the client application. std::vector ServerTimings; // query timings measured by the server. explicit TTestInfo(std::vector&& clientTimings, std::vector&& serverTimings); + void operator +=(const TTestInfo& other); + void operator /=(const ui32 count); }; class TQueryResultInfo { @@ -51,10 +55,10 @@ class TQueryResultInfo { class TQueryBenchmarkResult { private: - TString ErrorInfo; - TString YSONResult; - TQueryResultInfo QueryResult; - TDuration ServerTiming; + YDB_READONLY_DEF(TString, ErrorInfo); + YDB_READONLY_DEF(TString, YSONResult); + YDB_READONLY_DEF(TQueryResultInfo, QueryResult); + YDB_READONLY_DEF(TDuration, ServerTiming); TQueryBenchmarkResult() = default; public: static TQueryBenchmarkResult Result(const TString& yson, const TQueryResultInfo& queryResult, const TDuration& serverTiming) { @@ -69,20 +73,8 @@ class TQueryBenchmarkResult { result.ErrorInfo = error; return result; } - bool operator!() const { - return !!ErrorInfo; - } - const TString& GetErrorInfo() const { - return ErrorInfo; - } - TDuration GetServerTiming() const { - return ServerTiming; - } - const TString& GetYSONResult() const { - return YSONResult; - } - const TQueryResultInfo& GetQueryResult() const { - return QueryResult; + operator bool() const { + return !ErrorInfo; } }; diff --git a/ydb/public/lib/ydb_cli/commands/ydb_benchmark.cpp b/ydb/public/lib/ydb_cli/commands/ydb_benchmark.cpp index 28e2e89f7cf1..26d27feab9bd 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_benchmark.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_benchmark.cpp @@ -25,6 +25,9 @@ void TWorkloadCommandBenchmark::Config(TConfig& config) { config.Opts->AddLongOption("json", "Path to file to save json report to.\nJson report includes some metrics of queries, min and max time, stddev, etc. Has Solomon sensor format.") .DefaultValue("") .StoreResult(&JsonReportFileName); + config.Opts->AddLongOption("csv", "Path to file to save csv version of summary table.") + .DefaultValue("") + .StoreResult(&CsvReportFileName); config.Opts->AddLongOption("ministat", "Ministat report file name") .DefaultValue("") .StoreResult(&MiniStatFileName); @@ -72,7 +75,8 @@ void TWorkloadCommandBenchmark::Config(TConfig& config) { " Options: scan, generic\n" "scan - use scan queries;\n" "generic - use generic queries.") - .DefaultValue("scan").StoreResult(&QueryExecuterType); + .DefaultValue("generic").StoreResult(&QueryExecuterType); + config.Opts->AddLongOption('v', "verbose", "Verbose output").NoArgument().StoreValue(&VerboseLevel, 1); } TString TWorkloadCommandBenchmark::PatchQuery(const TStringBuf& original) const { @@ -104,29 +108,190 @@ bool TWorkloadCommandBenchmark::NeedRun(ui32 queryIdx) const { return true; } +namespace { + +TVector ColumnNames { + "Query #", + "ColdTime", + "Min", + "Max", + "Mean", + "Median", + "UnixBench", + "Std", + "RttMin", + "RttMax", + "RttAvg", + "SuccessCount", + "FailsCount", + "DiffsCount" +}; + +struct TTestInfoProduct { + double ColdTime = 1; + double Min = 1; + double Max = 1; + double RttMin = 1; + double RttMax = 1; + double RttMean = 1; + double Mean = 1; + double Median = 1; + double UnixBench = 1; + double Std = 0; + void operator *=(const BenchmarkUtils::TTestInfo& other) { + ColdTime *= other.ColdTime.MillisecondsFloat(); + Min *= other.Min.MillisecondsFloat(); + Max *= other.Max.MillisecondsFloat(); + RttMin *= other.RttMin.MillisecondsFloat(); + RttMax *= other.RttMax.MillisecondsFloat(); + Mean *= other.Mean; + Median *= other.Median; + UnixBench *= other.UnixBench.MillisecondsFloat(); + } + void operator ^= (ui32 count) { + ColdTime = pow(ColdTime, 1./count); + Min = pow(Min, 1./count); + Max = pow(Max, 1./count); + RttMin = pow(RttMin, 1./count); + RttMax = pow(RttMax, 1./count); + RttMean = pow(RttMean, 1./count); + Mean = pow(Mean, 1./count); + Median = pow(Median, 1./count); + UnixBench = pow(UnixBench, 1./count); + } +}; + +template +auto ValueToDouble(const T& value) { + return value; +} + +template<> +auto ValueToDouble(const double& value) { + return 0.001 * value; +} + +template<> +auto ValueToDouble(const TDuration& value) { + return 0.001 *value.MilliSeconds(); +} + +template::value> +struct TValueToTable { + static void Do(TPrettyTable::TRow& tableRow, ui32 index, const T& value) { + if (value) { + tableRow.Column(index, value); + } + } +}; + +template +struct TValueToTable{ + static void Do(TPrettyTable::TRow& tableRow, ui32 index, const T& value) { + tableRow.Column(index, Sprintf("%7.3f", value)); + } +}; + +template::value> +struct TValueToCsv { + static void Do(IOutputStream& csv, const T& value) { + csv << value; + } +}; + +template +struct TValueToCsv { + static void Do(IOutputStream& csv, const T& value) { + if (value) { + csv << value; + } + } +}; + +template::value> +struct TValueToJson { + static void Do(NJson::TJsonValue& json, ui32 index, ui32 queryN, const T& value) { + Y_UNUSED(json); + Y_UNUSED(index); + Y_UNUSED(queryN); + Y_UNUSED(value); + } +}; + +template +struct TValueToJson { + static void Do(NJson::TJsonValue& json, ui32 index, ui32 queryN, const T& value) { + json.AppendValue(BenchmarkUtils::GetSensorValue(ColumnNames[index], value, queryN)); + } +}; + + +template +void CollectField(TPrettyTable::TRow& tableRow, ui32 index, IOutputStream* csv, NJson::TJsonValue* json, TStringBuf rowName, const T& value) { + auto v = ValueToDouble(value); + TValueToTable::Do(tableRow, index, v); + if (csv) { + if (index) { + *csv << ","; + } + TValueToCsv::Do(*csv, v); + } + auto queryN = rowName; + if(json && queryN.SkipPrefix("Query")) { + TValueToJson::Do(*json, index, FromString(queryN), v); + } +} + +template +void CollectStats(TPrettyTable& table, IOutputStream* csv, NJson::TJsonValue* json, const TString& name, ui32 sCount, ui32 fCount, ui32 dCount, const T& testInfo) { + auto& row = table.AddRow(); + ui32 index = 0; + CollectField(row, index++, csv, json, name, name); + CollectField(row, index++, csv, json, name, testInfo.ColdTime); + CollectField(row, index++, csv, json, name, testInfo.Min); + CollectField(row, index++, csv, json, name, testInfo.Max); + CollectField(row, index++, csv, json, name, testInfo.Mean); + CollectField(row, index++, csv, json, name, testInfo.Median); + CollectField(row, index++, csv, json, name, testInfo.UnixBench); + CollectField(row, index++, csv, json, name, testInfo.Std); + CollectField(row, index++, csv, json, name, testInfo.RttMin); + CollectField(row, index++, csv, json, name, testInfo.RttMax); + CollectField(row, index++, csv, json, name, testInfo.RttMean); + CollectField(row, index++, csv, json, name, sCount); + CollectField(row, index++, csv, json, name, fCount); + CollectField(row, index++, csv, json, name, dCount); + if (csv) { + *csv << Endl; + } +} + +} + template bool TWorkloadCommandBenchmark::RunBench(TClient& client, NYdbWorkload::IWorkloadQueryGenerator& workloadGen) { using namespace BenchmarkUtils; TOFStream outFStream{OutFilePath}; - TPrettyTable statTable({ - "Query #", - "ColdTime", - "Min", - "Max", - "Mean", - "Median", - "Std", - "RttMin", - "RttMax", - "RttAvg" - }); + TPrettyTable statTable(ColumnNames); TStringStream report; report << "Results for " << IterationsCount << " iterations" << Endl; - NJson::TJsonValue jsonReport(NJson::JSON_ARRAY); - const bool collectJsonSensors = !JsonReportFileName.empty(); + THolder jsonReport; + if (JsonReportFileName) { + jsonReport = MakeHolder(NJson::JSON_ARRAY); + } const auto qtokens = workloadGen.GetWorkload(Type); - bool allOkay = true; + ui32 allSuccessQueries = 0; + ui32 someFailQueries = 0; + ui32 withDiffCount = 0; + THolder plansReport; + THolder csvReport; + if (CsvReportFileName) { + csvReport = MakeHolder(CsvReportFileName); + *csvReport << JoinSeq(",", ColumnNames) << Endl; + } + + TTestInfo sumInfo({}, {}); + TTestInfoProduct productInfo; std::map queryRuns; auto qIter = qtokens.cbegin(); @@ -148,14 +313,16 @@ bool TWorkloadCommandBenchmark::RunBench(TClient& client, NYdbWorkload::IWorkloa serverTimings.reserve(IterationsCount); Cout << Sprintf("Query%02u", queryN) << ":" << Endl; - Cerr << "Query text:\n" << Endl; - Cerr << query << Endl << Endl; + if (VerboseLevel > 0) { + Cout << "Query text:" << Endl; + Cout << query << Endl << Endl; + } ui32 successIteration = 0; ui32 failsCount = 0; ui32 diffsCount = 0; std::optional prevResult; - for (ui32 i = 0; i < IterationsCount * 10 && successIteration < IterationsCount; ++i) { + for (ui32 i = 0; i < IterationsCount; ++i) { auto t1 = TInstant::Now(); TQueryBenchmarkResult res = TQueryBenchmarkResult::Error("undefined"); try { @@ -166,7 +333,7 @@ bool TWorkloadCommandBenchmark::RunBench(TClient& client, NYdbWorkload::IWorkloa auto duration = TInstant::Now() - t1; Cout << "\titeration " << i << ":\t"; - if (!!res) { + if (res) { Cout << "ok\t" << duration << " seconds" << Endl; clientTimings.emplace_back(duration); serverTimings.emplace_back(res.GetServerTiming()); @@ -190,43 +357,34 @@ bool TWorkloadCommandBenchmark::RunBench(TClient& client, NYdbWorkload::IWorkloa Cout << "failed\t" << duration << " seconds" << Endl; Cerr << queryN << ": " << query << Endl << res.GetErrorInfo() << Endl; + Cerr << "Query text:" << Endl; + Cerr << query << Endl << Endl; Sleep(TDuration::Seconds(1)); } } - if (successIteration != IterationsCount) { - allOkay = false; - } - auto [inserted, success] = queryRuns.emplace(queryN, TTestInfo(std::move(clientTimings), std::move(serverTimings))); Y_ABORT_UNLESS(success); auto& testInfo = inserted->second; - statTable.AddRow() - .Column(0, Sprintf("Query%02u", queryN)) - .Column(1, Sprintf("%8.3f", 0.001 * testInfo.ColdTime.MilliSeconds())) - .Column(2, Sprintf("%7.3f", 0.001 * testInfo.Min.MilliSeconds())) - .Column(3, Sprintf("%7.3f", 0.001 * testInfo.Max.MilliSeconds())) - .Column(4, Sprintf("%8.3f", 0.001 * testInfo.Mean)) - .Column(5, Sprintf("%8.3f", 0.001 * testInfo.Median)) - .Column(6, Sprintf("%7.3f", 0.001 * testInfo.Std)) - .Column(7, Sprintf("%7.3f", 0.001 * testInfo.RttMin.MilliSeconds())) - .Column(8, Sprintf("%7.3f", 0.001 * testInfo.RttMax.MilliSeconds())) - .Column(9, Sprintf("%7.3f", 0.001 * testInfo.RttMean)); - - if (collectJsonSensors) { - jsonReport.AppendValue(GetSensorValue("ColdTime", testInfo.ColdTime, queryN)); - jsonReport.AppendValue(GetSensorValue("Min", testInfo.Min, queryN)); - jsonReport.AppendValue(GetSensorValue("Max", testInfo.Max, queryN)); - jsonReport.AppendValue(GetSensorValue("Mean", testInfo.Mean, queryN)); - jsonReport.AppendValue(GetSensorValue("Median", testInfo.Median, queryN)); - jsonReport.AppendValue(GetSensorValue("Std", testInfo.Std, queryN)); - jsonReport.AppendValue(GetSensorValue("RttMin", testInfo.RttMin, queryN)); - jsonReport.AppendValue(GetSensorValue("RttMax", testInfo.RttMax, queryN)); - jsonReport.AppendValue(GetSensorValue("RttMean", testInfo.RttMean, queryN)); - jsonReport.AppendValue(GetSensorValue("DiffsCount", diffsCount, queryN)); - jsonReport.AppendValue(GetSensorValue("FailsCount", failsCount, queryN)); - jsonReport.AppendValue(GetSensorValue("SuccessCount", successIteration, queryN)); + CollectStats(statTable, csvReport.Get(), jsonReport.Get(), Sprintf("Query%02u", queryN), successIteration, failsCount, diffsCount, testInfo); + if (successIteration != IterationsCount) { + ++someFailQueries; + } else { + ++allSuccessQueries; + sumInfo += testInfo; + productInfo *= testInfo; } + if (diffsCount) { + ++withDiffCount; + } + } + + if (allSuccessQueries) { + CollectStats(statTable, csvReport.Get(), jsonReport.Get(), "Sum", allSuccessQueries, someFailQueries, withDiffCount, sumInfo); + sumInfo /= allSuccessQueries; + CollectStats(statTable, csvReport.Get(), jsonReport.Get(), "Avg", allSuccessQueries, someFailQueries, withDiffCount, sumInfo); + productInfo ^= allSuccessQueries; + CollectStats(statTable, csvReport.Get(), jsonReport.Get(), "GAvg", allSuccessQueries, someFailQueries, withDiffCount, productInfo); } statTable.Print(report); @@ -254,14 +412,19 @@ bool TWorkloadCommandBenchmark::RunBench(TClient& client, NYdbWorkload::IWorkloa jStream.Finish(); } - if (collectJsonSensors) { + if (jsonReport) { TOFStream jStream{JsonReportFileName}; - NJson::WriteJson(&jStream, &jsonReport, /*formatOutput*/ true); + NJson::WriteJson(&jStream, jsonReport.Get(), /*formatOutput*/ true); jStream.Finish(); - Cout << "Report saved to " << JsonReportFileName << Endl; + Cout << "Json report saved to " << JsonReportFileName << Endl; + } + + if (csvReport) { + csvReport.Reset(); + Cout << "Summary table saved in CSV format to " << CsvReportFileName << Endl; } - return allOkay; + return !someFailQueries; } int TWorkloadCommandBenchmark::DoRun(NYdbWorkload::IWorkloadQueryGenerator& workloadGen, TConfig& /*config*/) { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_benchmark.h b/ydb/public/lib/ydb_cli/commands/ydb_benchmark.h index bc5fc8c11404..cd7fac3cdd61 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_benchmark.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_benchmark.h @@ -22,10 +22,12 @@ class TWorkloadCommandBenchmark final: public TWorkloadCommandBase { TString OutFilePath; ui32 IterationsCount; TString JsonReportFileName; + TString CsvReportFileName; TString MiniStatFileName; TSet QueriesToRun; TSet QueriesToSkip; TVector QuerySettings; + ui32 VerboseLevel = 0; }; } \ No newline at end of file