From 8af7fa8d3e93bd9c1fc107f103d4fe1e2b2be48f Mon Sep 17 00:00:00 2001 From: Daniil Demin Date: Tue, 15 Oct 2024 12:38:55 +0300 Subject: [PATCH] 24-3: Describe VIEW for YDB CLI (#9513) (#10356) --- ydb/core/driver_lib/run/run.cpp | 8 ++ ydb/core/driver_lib/run/ya.make | 1 + ydb/core/grpc_services/rpc_view.cpp | 94 +++++++++++++++++++ ydb/core/grpc_services/service_view.h | 12 +++ ydb/core/grpc_services/ya.make | 1 + ydb/public/api/grpc/draft/ya.make | 1 + ydb/public/api/grpc/draft/ydb_view_v1.proto | 10 ++ ydb/public/api/protos/draft/ydb_view.proto | 28 ++++++ ydb/public/api/protos/ya.make | 1 + .../ydb_cli/commands/ydb_service_scheme.cpp | 15 +++ .../lib/ydb_cli/commands/ydb_service_scheme.h | 4 + .../cpp/client/draft/ut/helpers/grpc_server.h | 16 ++++ .../ut/helpers/grpc_services/scripting.cpp | 22 +++++ .../ut/helpers/grpc_services/scripting.h | 16 ++++ .../draft/ut/helpers/grpc_services/view.cpp | 24 +++++ .../draft/ut/helpers/grpc_services/view.h | 18 ++++ .../sdk/cpp/client/draft/ut/helpers/ya.make | 13 +++ ydb/public/sdk/cpp/client/draft/ut/ya.make | 5 + .../ut/ydb_scripting_response_headers_ut.cpp | 30 ++++++ .../sdk/cpp/client/draft/ut/ydb_view_ut.cpp | 28 ++++++ ydb/public/sdk/cpp/client/draft/ya.make | 1 + .../ydb_scripting_response_headers_ut.cpp | 67 ------------- ydb/public/sdk/cpp/client/draft/ydb_view.cpp | 92 ++++++++++++++++++ ydb/public/sdk/cpp/client/draft/ydb_view.h | 62 ++++++++++++ .../sdk/cpp/client/ydb_proto/accessor.h | 3 + ydb/services/view/grpc_service.cpp | 34 +++++++ ydb/services/view/grpc_service.h | 15 +++ ydb/services/view/ya.make | 11 +++ ydb/services/ya.make | 1 + .../functional/ydb_cli/canondata/result.json | 3 + .../result.output | 4 + ydb/tests/functional/ydb_cli/conftest.py | 5 + .../functional/ydb_cli/test_ydb_scheme.py | 69 ++++++++++++++ ydb/tests/functional/ydb_cli/ya.make | 3 + 34 files changed, 650 insertions(+), 67 deletions(-) create mode 100644 ydb/core/grpc_services/rpc_view.cpp create mode 100644 ydb/core/grpc_services/service_view.h create mode 100644 ydb/public/api/grpc/draft/ydb_view_v1.proto create mode 100644 ydb/public/api/protos/draft/ydb_view.proto create mode 100644 ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_server.h create mode 100644 ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/scripting.cpp create mode 100644 ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/scripting.h create mode 100644 ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/view.cpp create mode 100644 ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/view.h create mode 100644 ydb/public/sdk/cpp/client/draft/ut/helpers/ya.make create mode 100644 ydb/public/sdk/cpp/client/draft/ut/ydb_scripting_response_headers_ut.cpp create mode 100644 ydb/public/sdk/cpp/client/draft/ut/ydb_view_ut.cpp delete mode 100644 ydb/public/sdk/cpp/client/draft/ydb_scripting_response_headers_ut.cpp create mode 100644 ydb/public/sdk/cpp/client/draft/ydb_view.cpp create mode 100644 ydb/public/sdk/cpp/client/draft/ydb_view.h create mode 100644 ydb/services/view/grpc_service.cpp create mode 100644 ydb/services/view/grpc_service.h create mode 100644 ydb/services/view/ya.make create mode 100644 ydb/tests/functional/ydb_cli/canondata/test_ydb_scheme.TestSchemeDescribe.test_describe_view/result.output create mode 100644 ydb/tests/functional/ydb_cli/conftest.py create mode 100644 ydb/tests/functional/ydb_cli/test_ydb_scheme.py diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index 5413287f4b96..2d9faa48bf1e 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -122,6 +122,7 @@ #include #include #include +#include #include @@ -600,6 +601,8 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { names["keyvalue"] = &hasKeyValue; TServiceCfg hasReplication = services.empty(); names["replication"] = &hasReplication; + TServiceCfg hasView = services.empty(); + names["view"] = &hasView; std::unordered_set enabled; for (const auto& name : services) { @@ -875,6 +878,11 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { grpcRequestProxies[0], hasReplication.IsRlAllowed())); } + if (hasView) { + server.AddService(new NGRpcService::TGRpcViewService(ActorSystem.Get(), Counters, + grpcRequestProxies[0], hasView.IsRlAllowed())); + } + if (ModuleFactories) { for (const auto& service : ModuleFactories->GrpcServiceFactory.Create(enabled, disabled, ActorSystem.Get(), Counters, grpcRequestProxies[0])) { server.AddService(service); diff --git a/ydb/core/driver_lib/run/ya.make b/ydb/core/driver_lib/run/ya.make index 7d7ea53e49e1..e479bf5b7b7e 100644 --- a/ydb/core/driver_lib/run/ya.make +++ b/ydb/core/driver_lib/run/ya.make @@ -172,6 +172,7 @@ PEERDIR( ydb/services/persqueue_v1 ydb/services/rate_limiter ydb/services/replication + ydb/services/view ydb/services/ydb ) diff --git a/ydb/core/grpc_services/rpc_view.cpp b/ydb/core/grpc_services/rpc_view.cpp new file mode 100644 index 000000000000..70172179716b --- /dev/null +++ b/ydb/core/grpc_services/rpc_view.cpp @@ -0,0 +1,94 @@ +#include "rpc_scheme_base.h" +#include "service_view.h" + +#include +#include +#include +#include +#include +#include + +namespace NKikimr::NGRpcService { + +using namespace Ydb; + +using TEvDescribeView = TGrpcRequestOperationCall; + +class TDescribeViewRPC : public TRpcSchemeRequestActor { + using TBase = TRpcSchemeRequestActor; + +public: + using TBase::TBase; + + void Bootstrap() { + DescribeScheme(); + } + + void PassAway() override { + TBase::PassAway(); + } + +private: + void DescribeScheme() { + auto ev = std::make_unique(); + SetAuthToken(ev, *Request_); + SetDatabase(ev.get(), *Request_); + ev->Record.MutableDescribePath()->SetPath(GetProtoRequest()->path()); + + Send(MakeTxProxyID(), ev.release()); + Become(&TDescribeViewRPC::StateDescribeScheme); + } + + STATEFN(StateDescribeScheme) { + switch (ev->GetTypeRewrite()) { + HFunc(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult, Handle); + default: + return TBase::StateWork(ev); + } + } + + void Handle(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev, const TActorContext& ctx) { + const auto& record = ev->Get()->GetRecord(); + const auto& desc = record.GetPathDescription(); + + if (record.HasReason()) { + Request_->RaiseIssue(NYql::TIssue(record.GetReason())); + } + + switch (record.GetStatus()) { + case NKikimrScheme::StatusSuccess: + if (desc.GetSelf().GetPathType() != NKikimrSchemeOp::EPathTypeView) { + auto message = TStringBuilder() << "Expected a view, but got: " << desc.GetSelf().GetPathType(); + Request_->RaiseIssue(NYql::TIssue(message)); + return Reply(StatusIds::SCHEME_ERROR, ctx); + } + + ConvertDirectoryEntry(desc.GetSelf(), Result_.mutable_self(), true); + Result_.set_query_text(desc.GetViewDescription().GetQueryText()); + + return ReplyWithResult(StatusIds::SUCCESS, Result_, ctx); + + case NKikimrScheme::StatusPathDoesNotExist: + case NKikimrScheme::StatusSchemeError: + return Reply(StatusIds::SCHEME_ERROR, ctx); + + case NKikimrScheme::StatusAccessDenied: + return Reply(StatusIds::UNAUTHORIZED, ctx); + + case NKikimrScheme::StatusNotAvailable: + return Reply(StatusIds::UNAVAILABLE, ctx); + + default: + return Reply(StatusIds::GENERIC_ERROR, ctx); + } + } + +private: + View::DescribeViewResult Result_; +}; + +void DoDescribeView(std::unique_ptr p, const IFacilityProvider& f) { + f.RegisterActor(new TDescribeViewRPC(p.release())); +} + +} diff --git a/ydb/core/grpc_services/service_view.h b/ydb/core/grpc_services/service_view.h new file mode 100644 index 000000000000..54021e0f3b18 --- /dev/null +++ b/ydb/core/grpc_services/service_view.h @@ -0,0 +1,12 @@ +#pragma once + +#include + +namespace NKikimr::NGRpcService { + +class IRequestOpCtx; +class IFacilityProvider; + +void DoDescribeView(std::unique_ptr p, const IFacilityProvider& f); + +} diff --git a/ydb/core/grpc_services/ya.make b/ydb/core/grpc_services/ya.make index 63344c31d9fb..0f58e4c2c20c 100644 --- a/ydb/core/grpc_services/ya.make +++ b/ydb/core/grpc_services/ya.make @@ -74,6 +74,7 @@ SRCS( rpc_stream_execute_yql_script.cpp rpc_whoami.cpp rpc_object_storage.cpp + rpc_view.cpp table_settings.cpp rpc_common/rpc_common_kqp_session.cpp diff --git a/ydb/public/api/grpc/draft/ya.make b/ydb/public/api/grpc/draft/ya.make index 80827825278b..33e7e205c3d1 100644 --- a/ydb/public/api/grpc/draft/ya.make +++ b/ydb/public/api/grpc/draft/ya.make @@ -16,6 +16,7 @@ SRCS( ydb_persqueue_v1.proto ydb_object_storage_v1.proto ydb_replication_v1.proto + ydb_view_v1.proto ydb_ymq_v1.proto ) diff --git a/ydb/public/api/grpc/draft/ydb_view_v1.proto b/ydb/public/api/grpc/draft/ydb_view_v1.proto new file mode 100644 index 000000000000..1d50326d16ff --- /dev/null +++ b/ydb/public/api/grpc/draft/ydb_view_v1.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package Ydb.View.V1; +option java_package = "com.yandex.ydb.view.v1"; + +import "ydb/public/api/protos/draft/ydb_view.proto"; + +service ViewService { + rpc DescribeView(View.DescribeViewRequest) returns (View.DescribeViewResponse); +} diff --git a/ydb/public/api/protos/draft/ydb_view.proto b/ydb/public/api/protos/draft/ydb_view.proto new file mode 100644 index 000000000000..bf88a51835e3 --- /dev/null +++ b/ydb/public/api/protos/draft/ydb_view.proto @@ -0,0 +1,28 @@ +syntax = "proto3"; +option cc_enable_arenas = true; + +package Ydb.View; +option java_package = "com.yandex.ydb.view"; + +import "ydb/public/api/protos/annotations/validation.proto"; +import "ydb/public/api/protos/ydb_operation.proto"; +import "ydb/public/api/protos/ydb_scheme.proto"; + +message DescribeViewRequest { + Ydb.Operations.OperationParams operation_params = 1; + // The path to the view. + string path = 2 [(required) = true]; +} + +message DescribeViewResponse { + // The result of the request will be inside the operation proto. + Ydb.Operations.Operation operation = 1; +} + +message DescribeViewResult { + // Description of a generic scheme object. + Ydb.Scheme.Entry self = 1; + + // View-specific fields. + string query_text = 2; +} diff --git a/ydb/public/api/protos/ya.make b/ydb/public/api/protos/ya.make index eb9031226aad..d8efc3029c68 100644 --- a/ydb/public/api/protos/ya.make +++ b/ydb/public/api/protos/ya.make @@ -17,6 +17,7 @@ SRCS( draft/ydb_maintenance.proto draft/ydb_object_storage.proto draft/ydb_replication.proto + draft/ydb_view.proto draft/ymq.proto draft/field_transformation.proto ydb_federation_discovery.proto diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp index d4eda58d6300..f3a117cbdaf3 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp @@ -179,6 +179,8 @@ int TCommandDescribe::PrintPathResponse(TDriver& driver, const NScheme::TDescrib return DescribeCoordinationNode(driver); case NScheme::ESchemeEntryType::Replication: return DescribeReplication(driver); + case NScheme::ESchemeEntryType::View: + return DescribeView(driver); default: return DescribeEntryDefault(entry); } @@ -491,6 +493,19 @@ int TCommandDescribe::DescribeReplication(const TDriver& driver) { return PrintDescription(this, OutputFormat, result, &TCommandDescribe::PrintReplicationResponsePretty); } +int TCommandDescribe::PrintViewResponsePretty(const NYdb::NView::TDescribeViewResult& result) const { + Cout << "\nQuery text:\n" << result.GetViewDescription().GetQueryText() << Endl; + return EXIT_SUCCESS; +} + +int TCommandDescribe::DescribeView(const TDriver& driver) { + NView::TViewClient client(driver); + auto result = client.DescribeView(Path, {}).ExtractValueSync(); + ThrowOnError(result); + + return PrintDescription(this, OutputFormat, result, &TCommandDescribe::PrintViewResponsePretty); +} + namespace { void PrintColumns(const NTable::TTableDescription& tableDescription) { if (!tableDescription.GetTableColumns().size()) { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.h b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.h index 070c65f299e9..fa5f6930f53f 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -69,6 +70,9 @@ class TCommandDescribe : public TYdbOperationCommand, public TCommandWithPath, p int DescribeReplication(const TDriver& driver); int PrintReplicationResponsePretty(const NYdb::NReplication::TDescribeReplicationResult& result) const; + int DescribeView(const TDriver& driver); + int PrintViewResponsePretty(const NYdb::NView::TDescribeViewResult& result) const; + template void PrintPermissionsIfNeeded(const TDescriptionType& description) const { if (ShowPermissions) { diff --git a/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_server.h b/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_server.h new file mode 100644 index 000000000000..4e3e544375d2 --- /dev/null +++ b/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_server.h @@ -0,0 +1,16 @@ +#pragma once + +#include +#include + +namespace NYdb { + +template +std::unique_ptr StartGrpcServer(const TString& address, TService& service) { + grpc::ServerBuilder builder; + builder.AddListeningPort(address, grpc::InsecureServerCredentials()); + builder.RegisterService(&service); + return builder.BuildAndStart(); +} + +} diff --git a/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/scripting.cpp b/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/scripting.cpp new file mode 100644 index 000000000000..d797ecce5ad4 --- /dev/null +++ b/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/scripting.cpp @@ -0,0 +1,22 @@ +#include "scripting.h" + +namespace NYdb::NScripting { + +grpc::Status TMockSlyDbProxy::ExecuteYql( + grpc::ServerContext* context, + const Ydb::Scripting::ExecuteYqlRequest* request, + Ydb::Scripting::ExecuteYqlResponse* response +) { + context->AddInitialMetadata("key", "value"); + Y_UNUSED(request); + + // Just to make sdk core happy + auto* op = response->mutable_operation(); + op->set_ready(true); + op->set_status(Ydb::StatusIds::SUCCESS); + op->mutable_result(); + + return grpc::Status::OK; +} + +} diff --git a/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/scripting.h b/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/scripting.h new file mode 100644 index 000000000000..6de9e62ed3bd --- /dev/null +++ b/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/scripting.h @@ -0,0 +1,16 @@ +#pragma once + +#include + +namespace NYdb::NScripting { + +class TMockSlyDbProxy : public Ydb::Scripting::V1::ScriptingService::Service +{ +public: + grpc::Status ExecuteYql( + grpc::ServerContext* context, + const Ydb::Scripting::ExecuteYqlRequest* request, + Ydb::Scripting::ExecuteYqlResponse* response) override; +}; + +} diff --git a/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/view.cpp b/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/view.cpp new file mode 100644 index 000000000000..03db2dd68f96 --- /dev/null +++ b/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/view.cpp @@ -0,0 +1,24 @@ +#include "view.h" + +namespace NYdb::NView { + +grpc::Status TViewDummyService::DescribeView( + grpc::ServerContext* context, + const Ydb::View::DescribeViewRequest* request, + Ydb::View::DescribeViewResponse* response +) { + Y_UNUSED(context); + Y_UNUSED(request); + + auto* op = response->mutable_operation(); + op->set_ready(true); + op->set_status(Ydb::StatusIds::SUCCESS); + + Ydb::View::DescribeViewResult describeResult; + describeResult.set_query_text(DummyQueryText); + op->mutable_result()->PackFrom(describeResult); + + return grpc::Status::OK; +} + +} diff --git a/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/view.h b/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/view.h new file mode 100644 index 000000000000..704573dbb2d1 --- /dev/null +++ b/ydb/public/sdk/cpp/client/draft/ut/helpers/grpc_services/view.h @@ -0,0 +1,18 @@ +#pragma once + +#include + +namespace NYdb::NView { + +constexpr const char* DummyQueryText = "select 42"; + +class TViewDummyService : public Ydb::View::V1::ViewService::Service +{ +public: + grpc::Status DescribeView( + grpc::ServerContext* context, + const Ydb::View::DescribeViewRequest* request, + Ydb::View::DescribeViewResponse* response) override; +}; + +} diff --git a/ydb/public/sdk/cpp/client/draft/ut/helpers/ya.make b/ydb/public/sdk/cpp/client/draft/ut/helpers/ya.make new file mode 100644 index 000000000000..2379d6e15cfa --- /dev/null +++ b/ydb/public/sdk/cpp/client/draft/ut/helpers/ya.make @@ -0,0 +1,13 @@ +LIBRARY() + +PEERDIR( + ydb/public/api/grpc + ydb/public/api/grpc/draft +) + +SRCS( + grpc_services/scripting.cpp + grpc_services/view.cpp +) + +END() diff --git a/ydb/public/sdk/cpp/client/draft/ut/ya.make b/ydb/public/sdk/cpp/client/draft/ut/ya.make index c7dd2ee98903..9bb1caf413cf 100644 --- a/ydb/public/sdk/cpp/client/draft/ut/ya.make +++ b/ydb/public/sdk/cpp/client/draft/ut/ya.make @@ -11,8 +11,13 @@ ENDIF() FORK_SUBTESTS() +PEERDIR( + ydb/public/sdk/cpp/client/draft/ut/helpers +) + SRCS( ydb_scripting_response_headers_ut.cpp + ydb_view_ut.cpp ) END() diff --git a/ydb/public/sdk/cpp/client/draft/ut/ydb_scripting_response_headers_ut.cpp b/ydb/public/sdk/cpp/client/draft/ut/ydb_scripting_response_headers_ut.cpp new file mode 100644 index 000000000000..6f9bc139f566 --- /dev/null +++ b/ydb/public/sdk/cpp/client/draft/ut/ydb_scripting_response_headers_ut.cpp @@ -0,0 +1,30 @@ +#include "helpers/grpc_server.h" +#include "helpers/grpc_services/scripting.h" + +#include + +#include + +using namespace NYdb; +using namespace NYdb::NScripting; + +Y_UNIT_TEST_SUITE(ResponseHeaders) { + Y_UNIT_TEST(PassHeader) { + TMockSlyDbProxy slyDbProxy; + + TString addr = "localhost:2135"; + + auto server = StartGrpcServer(addr, slyDbProxy); + + auto config = TDriverConfig() + .SetEndpoint(addr); + TDriver driver(config); + TScriptingClient client(driver); + + auto result = client.ExecuteYqlScript("SMTH").GetValueSync(); + auto metadata = result.GetResponseMetadata(); + + UNIT_ASSERT(metadata.find("key") != metadata.end()); + UNIT_ASSERT_VALUES_EQUAL(metadata.find("key")->second, "value"); + } +} diff --git a/ydb/public/sdk/cpp/client/draft/ut/ydb_view_ut.cpp b/ydb/public/sdk/cpp/client/draft/ut/ydb_view_ut.cpp new file mode 100644 index 000000000000..e5a87e0ee27b --- /dev/null +++ b/ydb/public/sdk/cpp/client/draft/ut/ydb_view_ut.cpp @@ -0,0 +1,28 @@ +#include "helpers/grpc_server.h" +#include "helpers/grpc_services/view.h" + +#include + +#include + +using namespace NYdb; +using namespace NYdb::NView; + +Y_UNIT_TEST_SUITE(ViewClient) { + Y_UNIT_TEST(Basic) { + TString addr = "localhost:2135"; + TViewDummyService viewService; + + auto server = StartGrpcServer(addr, viewService); + + auto config = TDriverConfig().SetEndpoint(addr); + TDriver driver(config); + TViewClient client(driver); + + auto result = client.DescribeView("any").ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + auto queryText = result.GetViewDescription().GetQueryText(); + UNIT_ASSERT_STRINGS_EQUAL(queryText, DummyQueryText); + } +} diff --git a/ydb/public/sdk/cpp/client/draft/ya.make b/ydb/public/sdk/cpp/client/draft/ya.make index 6a7803c48991..9923f0c34b5c 100644 --- a/ydb/public/sdk/cpp/client/draft/ya.make +++ b/ydb/public/sdk/cpp/client/draft/ya.make @@ -4,6 +4,7 @@ SRCS( ydb_dynamic_config.cpp ydb_replication.cpp ydb_scripting.cpp + ydb_view.cpp ) GENERATE_ENUM_SERIALIZATION(ydb_replication.h) diff --git a/ydb/public/sdk/cpp/client/draft/ydb_scripting_response_headers_ut.cpp b/ydb/public/sdk/cpp/client/draft/ydb_scripting_response_headers_ut.cpp deleted file mode 100644 index 1d67d6e58016..000000000000 --- a/ydb/public/sdk/cpp/client/draft/ydb_scripting_response_headers_ut.cpp +++ /dev/null @@ -1,67 +0,0 @@ - -#include -#include -#include - -#include -#include -#include - -#include -#include - -using namespace NYdb; -using namespace NYdb::NScripting; - -namespace { - -template -std::unique_ptr StartGrpcServer(const TString& address, TService& service) { - grpc::ServerBuilder builder; - builder.AddListeningPort(address, grpc::InsecureServerCredentials()); - builder.RegisterService(&service); - return builder.BuildAndStart(); -} - -class TMockSlyDbProxy : public Ydb::Scripting::V1::ScriptingService::Service -{ -public: - grpc::Status ExecuteYql( - grpc::ServerContext* context, - const Ydb::Scripting::ExecuteYqlRequest* request, - Ydb::Scripting::ExecuteYqlResponse* response) override { - context->AddInitialMetadata("key", "value"); - Y_UNUSED(request); - - // Just to make sdk core happy - auto* op = response->mutable_operation(); - op->set_ready(true); - op->set_status(Ydb::StatusIds::SUCCESS); - op->mutable_result(); - - return grpc::Status::OK; - } -}; - -} - -Y_UNIT_TEST_SUITE(ResponseHeaders) { - Y_UNIT_TEST(PassHeader) { - TMockSlyDbProxy slyDbProxy; - - TString addr = "localhost:2135"; - - auto server = StartGrpcServer(addr, slyDbProxy); - - auto config = TDriverConfig() - .SetEndpoint(addr); - TDriver driver(config); - TScriptingClient client(driver); - - auto result = client.ExecuteYqlScript("SMTH").GetValueSync(); - auto metadata = result.GetResponseMetadata(); - - UNIT_ASSERT(metadata.find("key") != metadata.end()); - UNIT_ASSERT_VALUES_EQUAL(metadata.find("key")->second, "value"); - } -} diff --git a/ydb/public/sdk/cpp/client/draft/ydb_view.cpp b/ydb/public/sdk/cpp/client/draft/ydb_view.cpp new file mode 100644 index 000000000000..d58616d15e54 --- /dev/null +++ b/ydb/public/sdk/cpp/client/draft/ydb_view.cpp @@ -0,0 +1,92 @@ +#include "ydb_view.h" + +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include +#include +#include +#include + +namespace NYdb { +namespace NView { + +TViewDescription::TViewDescription(const Ydb::View::DescribeViewResult& desc) + : QueryText_(desc.query_text()) +{ +} + +const TString& TViewDescription::GetQueryText() const { + return QueryText_; +} + +TDescribeViewResult::TDescribeViewResult(TStatus&& status, Ydb::View::DescribeViewResult&& desc) + : NScheme::TDescribePathResult(std::move(status), desc.self()) + , Proto_(std::make_unique(std::move(desc))) +{ +} + +TViewDescription TDescribeViewResult::GetViewDescription() const { + return TViewDescription(*Proto_); +} + +const Ydb::View::DescribeViewResult& TDescribeViewResult::GetProto() const { + return *Proto_; +} + +class TViewClient::TImpl : public TClientImplCommon { +public: + TImpl(std::shared_ptr&& connections, const TCommonClientSettings& settings) + : TClientImplCommon(std::move(connections), settings) + { + } + + TAsyncDescribeViewResult DescribeView(const TString& path, const TDescribeViewSettings& settings) { + using namespace Ydb::View; + + auto request = MakeOperationRequest(settings); + request.set_path(path); + + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + DescribeViewResult result; + if (any) { + any->UnpackTo(&result); + } + + TDescribeViewResult val(TStatus(std::move(status)), std::move(result)); + promise.SetValue(std::move(val)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &V1::ViewService::Stub::AsyncDescribeView, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + +}; + +TViewClient::TViewClient(const TDriver& driver, const TCommonClientSettings& settings) + : Impl_(std::make_shared(CreateInternalInterface(driver), settings)) +{ +} + +TAsyncDescribeViewResult TViewClient::DescribeView(const TString& path, const TDescribeViewSettings& settings) { + return Impl_->DescribeView(path, settings); +} + +} // NView + +const Ydb::View::DescribeViewResult& TProtoAccessor::GetProto(const NView::TDescribeViewResult& result) { + return result.GetProto(); +} + +} // NYdb diff --git a/ydb/public/sdk/cpp/client/draft/ydb_view.h b/ydb/public/sdk/cpp/client/draft/ydb_view.h new file mode 100644 index 000000000000..6f9908e90414 --- /dev/null +++ b/ydb/public/sdk/cpp/client/draft/ydb_view.h @@ -0,0 +1,62 @@ +#pragma once + +#include +#include + +namespace Ydb::View { + class DescribeViewResult; +} + +namespace NYdb { + class TProtoAccessor; +} + +namespace NYql { + class TIssues; +} + +namespace NYdb::NView { + +class TDescribeViewResult; +using TAsyncDescribeViewResult = NThreading::TFuture; + +struct TDescribeViewSettings : public TOperationRequestSettings { + using TSelf = TDescribeViewSettings; +}; + +class TViewDescription { +public: + explicit TViewDescription(const Ydb::View::DescribeViewResult& desc); + + const TString& GetQueryText() const; + +private: + TString QueryText_; +}; + +class TDescribeViewResult : public NScheme::TDescribePathResult { + friend class NYdb::TProtoAccessor; + const Ydb::View::DescribeViewResult& GetProto() const; + +public: + TDescribeViewResult(TStatus&& status, Ydb::View::DescribeViewResult&& desc); + TViewDescription GetViewDescription() const; + +private: + std::unique_ptr Proto_; +}; + +class TViewClient { + class TImpl; + +public: + TViewClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); + + TAsyncDescribeViewResult DescribeView(const TString& path, + const TDescribeViewSettings& settings = TDescribeViewSettings()); + +private: + std::shared_ptr Impl_; +}; + +} // namespace NYdb::NView diff --git a/ydb/public/sdk/cpp/client/ydb_proto/accessor.h b/ydb/public/sdk/cpp/client/ydb_proto/accessor.h index b69f26e3d2ff..7c609bb01176 100644 --- a/ydb/public/sdk/cpp/client/ydb_proto/accessor.h +++ b/ydb/public/sdk/cpp/client/ydb_proto/accessor.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -11,6 +12,7 @@ #include #include +#include #include #include #include @@ -48,6 +50,7 @@ class TProtoAccessor { static const Ydb::Monitoring::SelfCheckResult& GetProto(const NYdb::NMonitoring::TSelfCheckResult& selfCheckResult); static const Ydb::Coordination::DescribeNodeResult& GetProto(const NYdb::NCoordination::TNodeDescription &describeNodeResult); static const Ydb::Replication::DescribeReplicationResult& GetProto(const NYdb::NReplication::TDescribeReplicationResult& desc); + static const Ydb::View::DescribeViewResult& GetProto(const NYdb::NView::TDescribeViewResult& desc); static NTable::TQueryStats FromProto(const Ydb::TableStats::QueryStats& queryStats); static NTable::TTableDescription FromProto(const Ydb::Table::CreateTableRequest& request); diff --git a/ydb/services/view/grpc_service.cpp b/ydb/services/view/grpc_service.cpp new file mode 100644 index 000000000000..03367ec5a8a0 --- /dev/null +++ b/ydb/services/view/grpc_service.cpp @@ -0,0 +1,34 @@ +#include "grpc_service.h" + +#include +#include +#include + +namespace NKikimr::NGRpcService { + +void TGRpcViewService::SetupIncomingRequests(NYdbGrpc::TLoggerPtr logger) { + using namespace Ydb::View; + + auto getCounterBlock = CreateCounterCb(Counters_, ActorSystem_); + + MakeIntrusive>( + this, &Service_, CQ_, + [this](NYdbGrpc::IRequestContextBase* ctx) { + NGRpcService::ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); + ActorSystem_->Send( + GRpcRequestProxyId_, + new TGrpcRequestOperationCall( + ctx, &DoDescribeView, + TRequestAuxSettings{RLSWITCH(TRateLimiterMode::Rps), nullptr} + ) + ); + }, + &V1::ViewService::AsyncService::RequestDescribeView, + "DescribeView", logger, getCounterBlock("view", "DescribeView") + )->Run(); +} + +} diff --git a/ydb/services/view/grpc_service.h b/ydb/services/view/grpc_service.h new file mode 100644 index 000000000000..695fc70f5c2e --- /dev/null +++ b/ydb/services/view/grpc_service.h @@ -0,0 +1,15 @@ +#pragma once + +#include +#include + +namespace NKikimr::NGRpcService { + +class TGRpcViewService: public TGrpcServiceBase { +public: + using TGrpcServiceBase::TGrpcServiceBase; +private: + void SetupIncomingRequests(NYdbGrpc::TLoggerPtr logger); +}; + +} diff --git a/ydb/services/view/ya.make b/ydb/services/view/ya.make new file mode 100644 index 000000000000..94e5d9031c0c --- /dev/null +++ b/ydb/services/view/ya.make @@ -0,0 +1,11 @@ +LIBRARY() + +SRCS( + grpc_service.cpp +) + +PEERDIR( + ydb/core/grpc_services +) + +END() diff --git a/ydb/services/ya.make b/ydb/services/ya.make index 636feb611120..c7ffdf034239 100644 --- a/ydb/services/ya.make +++ b/ydb/services/ya.make @@ -20,6 +20,7 @@ RECURSE( persqueue_v1 rate_limiter replication + view ydb ymq ) diff --git a/ydb/tests/functional/ydb_cli/canondata/result.json b/ydb/tests/functional/ydb_cli/canondata/result.json index 5dacdfc4b348..04d83f052824 100644 --- a/ydb/tests/functional/ydb_cli/canondata/result.json +++ b/ydb/tests/functional/ydb_cli/canondata/result.json @@ -227,6 +227,9 @@ "test_ydb_impex.TestImpex.test_stdin[tsv-additional_args3-row]": { "uri": "file://test_ydb_impex.TestImpex.test_stdin_tsv-additional_args3-row_/result.output" }, + "test_ydb_scheme.TestSchemeDescribe.test_describe_view": { + "uri": "file://test_ydb_scheme.TestSchemeDescribe.test_describe_view/result.output" + }, "test_ydb_scripting.TestExecuteScriptFromStdinWithWideOutput.test_wide_table": { "uri": "file://test_ydb_scripting.TestExecuteScriptFromStdinWithWideOutput.test_wide_table/result.output" }, diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scheme.TestSchemeDescribe.test_describe_view/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scheme.TestSchemeDescribe.test_describe_view/result.output new file mode 100644 index 000000000000..729b29bcdccc --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scheme.TestSchemeDescribe.test_describe_view/result.output @@ -0,0 +1,4 @@ + view + +Query text: +SELECT 42 diff --git a/ydb/tests/functional/ydb_cli/conftest.py b/ydb/tests/functional/ydb_cli/conftest.py new file mode 100644 index 000000000000..b6c9a98e3f48 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/conftest.py @@ -0,0 +1,5 @@ +# XXX: setting of pytest_plugins should work if specified directly in test modules +# but somehow it does not +# +# for ydb_{cluster, database, ...} fixture family +pytest_plugins = 'ydb.tests.library.harness.ydb_fixtures' diff --git a/ydb/tests/functional/ydb_cli/test_ydb_scheme.py b/ydb/tests/functional/ydb_cli/test_ydb_scheme.py new file mode 100644 index 000000000000..853abe2ede60 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/test_ydb_scheme.py @@ -0,0 +1,69 @@ +import json +import os +import pytest + +from ydb.tests.library.common import yatest_common +from ydb.tests.oss.canonical import set_canondata_root + +CLUSTER_CONFIG = dict(extra_feature_flags=["enable_views"], extra_grpc_services=["view"]) + + +def bin_from_env(var): + if os.getenv(var): + return yatest_common.binary_path(os.getenv(var)) + raise RuntimeError(f"{var} environment variable is not specified") + + +def ydb_bin(): + return bin_from_env("YDB_CLI_BINARY") + + +def canonical_result(output_result, tmp_path): + output_path = tmp_path / "result.output" + with output_path.open("w") as f: + f.write(output_result) + return yatest_common.canonical_file(output_path, local=True, universal_lines=True) + + +def execute_ydb_cli_command(node, database, args, stdin=None): + execution = yatest_common.execute( + [ydb_bin(), "--endpoint", f"grpc://{node.host}:{node.grpc_port}", "--database", database] + args, stdin=stdin + ) + return execution.std_out.decode("utf-8") + + +def create_view(session, view, query="SELECT 42"): + session.execute_scheme( + f""" + CREATE VIEW {view} WITH security_invoker = TRUE AS {query}; + """ + ) + + +class TestSchemeDescribe: + @pytest.fixture(autouse=True, scope="function") + def init_test(self, tmp_path): + self.tmp_path = tmp_path + set_canondata_root("ydb/tests/functional/ydb_cli/canondata") + + def test_describe_view(self, ydb_cluster, ydb_database, ydb_client_session): + database_path = ydb_database + session_pool = ydb_client_session(database_path) + with session_pool.checkout() as session: + view = "view" + create_view(session, view) + output = execute_ydb_cli_command(ydb_cluster.nodes[1], database_path, ["scheme", "describe", view]) + return canonical_result(output, self.tmp_path) + + def test_describe_view_json(self, ydb_cluster, ydb_database, ydb_client_session): + database_path = ydb_database + session_pool = ydb_client_session(database_path) + with session_pool.checkout() as session: + view = "view" + query = "select 1" + create_view(session, view, query) + output = execute_ydb_cli_command( + ydb_cluster.nodes[1], database_path, ["scheme", "describe", "--format", "proto-json-base64", view] + ) + description = output.splitlines()[1] + assert json.loads(description)["query_text"] == query diff --git a/ydb/tests/functional/ydb_cli/ya.make b/ydb/tests/functional/ydb_cli/ya.make index 1e4310e5e7fc..5df08cab9885 100644 --- a/ydb/tests/functional/ydb_cli/ya.make +++ b/ydb/tests/functional/ydb_cli/ya.make @@ -1,16 +1,19 @@ PY3TEST() TEST_SRCS( + conftest.py test_ydb_backup.py test_ydb_table.py test_ydb_scripting.py test_ydb_impex.py test_ydb_flame_graph.py + test_ydb_scheme.py ) ENV(YDB_DRIVER_BINARY="ydb/apps/ydbd/ydbd") ENV(YDB_CLI_BINARY="ydb/apps/ydb/ydb") ENV(YDB_ENABLE_COLUMN_TABLES="true") + TIMEOUT(600) SIZE(MEDIUM)