Skip to content

Commit

Permalink
Merge fe06452 into b12c122
Browse files Browse the repository at this point in the history
  • Loading branch information
GrigoriyPA authored Sep 6, 2024
2 parents b12c122 + fe06452 commit 0556e28
Show file tree
Hide file tree
Showing 10 changed files with 209 additions and 0 deletions.
40 changes: 40 additions & 0 deletions ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -7212,6 +7212,46 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString());
UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "Classifier with name MyResourcePoolClassifier not found in database /Root");
}

Y_UNIT_TEST(DisableMetadataObjectsOnServerless) {
auto ydb = NWorkload::TYdbSetupSettings()
.CreateSampleTenants(true)
.EnableMetadataObjectsOnServerless(false)
.Create();

auto checkDisabled = [](const auto& result) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::GENERIC_ERROR, result.GetIssues().ToString());
UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "Objects SECRET are disabled for serverless domains. Please contact your system administrator to enable it");
};

const auto& createSql = "CREATE OBJECT MySecretObject (TYPE SECRET) WITH (value=\"qwerty\");";
const auto& alterSql = "ALTER OBJECT MySecretObject (TYPE SECRET) SET value = \"abcde\";";
const auto& upsertSql = "UPSERT OBJECT MySecretObject (TYPE SECRET) WITH value = \"edcba\";";
const auto& dropSql = "DROP OBJECT MySecretObject (TYPE SECRET);";

auto settings = NWorkload::TQueryRunnerSettings().PoolId("");

// Dedicated, enabled
settings.Database(ydb->GetSettings().GetDedicatedTenantName()).NodeIndex(1);
NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(createSql, settings));
NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(alterSql, settings));
NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(upsertSql, settings));
NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropSql, settings));

// Shared, enabled
settings.Database(ydb->GetSettings().GetSharedTenantName()).NodeIndex(2);
NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(createSql, settings));
NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(alterSql, settings));
NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(upsertSql, settings));
NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropSql, settings));

// Serverless, disabled
settings.Database(ydb->GetSettings().GetServerlessTenantName()).NodeIndex(2);
checkDisabled(ydb->ExecuteQuery(createSql, settings));
checkDisabled(ydb->ExecuteQuery(alterSql, settings));
checkDisabled(ydb->ExecuteQuery(upsertSql, settings));
NWorkload::TSampleQueries::CheckSuccess(ydb->ExecuteQuery(dropSql, settings));
}
}

Y_UNIT_TEST_SUITE(KqpOlapScheme) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,7 @@ class TWorkloadServiceYdbSetup : public IYdbSetup {
TAppConfig appConfig;
appConfig.MutableFeatureFlags()->SetEnableResourcePools(Settings_.EnableResourcePools_);
appConfig.MutableFeatureFlags()->SetEnableResourcePoolsOnServerless(Settings_.EnableResourcePoolsOnServerless_);
appConfig.MutableFeatureFlags()->SetEnableMetadataObjectsOnServerless(Settings_.EnableMetadataObjectsOnServerless_);
appConfig.MutableFeatureFlags()->SetEnableResourcePoolsCounters(true);

return appConfig;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ struct TYdbSetupSettings {
FLUENT_SETTING_DEFAULT(bool, CreateSampleTenants, false);
FLUENT_SETTING_DEFAULT(bool, EnableResourcePools, true);
FLUENT_SETTING_DEFAULT(bool, EnableResourcePoolsOnServerless, false);
FLUENT_SETTING_DEFAULT(bool, EnableMetadataObjectsOnServerless, true);

// Default pool settings
FLUENT_SETTING_DEFAULT(TString, PoolId, "sample_pool_id");
Expand Down
1 change: 1 addition & 0 deletions ydb/core/protos/feature_flags.proto
Original file line number Diff line number Diff line change
Expand Up @@ -157,4 +157,5 @@ message TFeatureFlags {
optional bool EnableAlterShardingInColumnShard = 138 [default = false];
optional bool EnablePgSyntax = 139 [default = true];
optional bool EnableTieringInColumnShard = 140 [default = false];
optional bool EnableMetadataObjectsOnServerless = 141 [default = true];
}
1 change: 1 addition & 0 deletions ydb/core/testlib/basics/feature_flags.h
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ class TTestFeatureFlagsHolder {
FEATURE_FLAG_SETTER(EnableGranularTimecast)
FEATURE_FLAG_SETTER(EnablePgSyntax)
FEATURE_FLAG_SETTER(EnableTieringInColumnShard)
FEATURE_FLAG_SETTER(EnableMetadataObjectsOnServerless)

#undef FEATURE_FLAG_SETTER
};
Expand Down
26 changes: 26 additions & 0 deletions ydb/services/metadata/manager/alter_impl.h
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#pragma once
#include "abstract.h"
#include "fetch_database.h"
#include "modification_controller.h"
#include "preparation_controller.h"
#include "restore.h"
Expand Down Expand Up @@ -111,6 +112,7 @@ class TModificationActorImpl: public NActors::TActorBootstrapped<TModificationAc
hFunc(NRequest::TEvRequestFailed, Handle);
hFunc(TEvRestoreProblem, Handle);
hFunc(TEvAlterPreparationProblem, Handle);
hFunc(TEvFetchDatabaseResponse, Handle);
default:
break;
}
Expand All @@ -126,6 +128,30 @@ class TModificationActorImpl: public NActors::TActorBootstrapped<TModificationAc
return this->PassAway();
}

if (!AppData()->FeatureFlags.GetEnableMetadataObjectsOnServerless() && Context.GetActivityType() != IOperationsManager::EActivityType::Drop) {
TBase::Register(CreateDatabaseFetcherActor(Context.GetExternalData().GetDatabase()));
} else {
CreateSession();
}
}

void Handle(TEvFetchDatabaseResponse::TPtr& ev) {
TString errorMessage;
if (const auto& errorString = ev->Get()->GetErrorString()) {
errorMessage = TStringBuilder() << "Cannot fetch database '" << Context.GetExternalData().GetDatabase() << "': " << *errorString;
} else if (ev->Get()->GetServerless()) {
errorMessage = TStringBuilder() << "Objects " << TObject::GetTypeId() << " are disabled for serverless domains. Please contact your system administrator to enable it";
}

if (errorMessage) {
auto g = TBase::PassAwayGuard();
ExternalController->OnAlteringProblem(errorMessage);
} else {
CreateSession();
}
}

void CreateSession() const {
TBase::Register(new NRequest::TYDBCallbackRequest<NRequest::TDialogCreateSession>(
NRequest::TDialogCreateSession::TRequest(), UserToken, TBase::SelfId()));
}
Expand Down
1 change: 1 addition & 0 deletions ydb/services/metadata/manager/common.h
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ enum EEvents {
EvAlterProblem,
EvAlterPreparationFinished,
EvAlterPreparationProblem,
EvFetchDatabaseResponse,
EvEnd
};
static_assert(EEvents::EvEnd < EventSpaceEnd(TKikimrEvents::ES_METADATA_MANAGER), "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_METADATA_MANAGER)");
Expand Down
112 changes: 112 additions & 0 deletions ydb/services/metadata/manager/fetch_database.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
#include "fetch_database.h"

#include <library/cpp/retry/retry_policy.h>

#include <ydb/core/base/appdata_fwd.h>

#include <ydb/library/table_creator/table_creator.h>


namespace NKikimr::NMetadata::NModifications {

namespace {

class TDatabaseFetcherActor : public TActorBootstrapped<TDatabaseFetcherActor> {
using TBase = TActorBootstrapped<TDatabaseFetcherActor>;
using TRetryPolicy = IRetryPolicy<>;

public:
explicit TDatabaseFetcherActor(const TString& database)
: Database(database)
{}

void Registered(TActorSystem* sys, const TActorId& owner) override {
TBase::Registered(sys, owner);
Owner = owner;
}

void Bootstrap() {
StartRequest();
Become(&TDatabaseFetcherActor::StateFunc);
}

void Handle(TEvents::TEvUndelivered::TPtr& ev) {
if (ev->Get()->Reason == NActors::TEvents::TEvUndelivered::ReasonActorUnknown && ScheduleRetry()) {
return;
}

Reply("Scheme cache is unavailable");
}

void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) {
const auto& results = ev->Get()->Request->ResultSet;
Y_ABORT_UNLESS(results.size() == 1);

const auto& result = results[0];
if (result.DomainInfo) {
Serverless = result.DomainInfo->IsServerless();
Reply();
return;
}

if (result.Status == NSchemeCache::TSchemeCacheNavigate::EStatus::LookupError && ScheduleRetry()) {
return;
}

Reply(TStringBuilder() << "Failed to fetch database info: " << result.Status);
}

STRICT_STFUNC(StateFunc,
sFunc(TEvents::TEvWakeup, StartRequest);
hFunc(TEvents::TEvUndelivered, Handle);
hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle);
)

private:
void StartRequest() {
auto event = NTableCreator::BuildSchemeCacheNavigateRequest(
{{}},
Database ? Database : AppData()->TenantName,
MakeIntrusive<NACLib::TUserToken>(BUILTIN_ACL_METADATA, TVector<NACLib::TSID>{})
);
event->ResultSet[0].Operation = NSchemeCache::TSchemeCacheNavigate::OpPath;
Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(event.Release()), IEventHandle::FlagTrackDelivery);
}

bool ScheduleRetry() {
if (!RetryState) {
RetryState = TRetryPolicy::GetFixedIntervalPolicy(
[](){return ERetryErrorClass::ShortRetry;}
, TDuration::MilliSeconds(100)
, TDuration::MilliSeconds(500)
, 100
)->CreateRetryState();;
}

if (const auto delay = RetryState->GetNextRetryDelay()) {
this->Schedule(*delay, new TEvents::TEvWakeup());
return true;
}

return false;
}

void Reply(const std::optional<TString>& errorMessage = std::nullopt) {
Send(Owner, new TEvFetchDatabaseResponse(Serverless, errorMessage));
PassAway();
}

private:
const TString Database;
TActorId Owner;
TRetryPolicy::IRetryState::TPtr RetryState;
bool Serverless = false;
};

} // anonymous namespace

IActor* CreateDatabaseFetcherActor(const TString& database) {
return new TDatabaseFetcherActor(database);
}

} // NKikimr::NMetadata::NModifications
24 changes: 24 additions & 0 deletions ydb/services/metadata/manager/fetch_database.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
#pragma once

#include "common.h"

#include <ydb/library/accessor/accessor.h>


namespace NKikimr::NMetadata::NModifications {

class TEvFetchDatabaseResponse : public TEventLocal<TEvFetchDatabaseResponse, EvFetchDatabaseResponse> {
private:
YDB_READONLY_DEF(bool, Serverless);
YDB_READONLY_DEF(std::optional<TString>, ErrorString);

public:
TEvFetchDatabaseResponse(bool serverless, const std::optional<TString>& errorString)
: Serverless(serverless)
, ErrorString(errorString)
{}
};

IActor* CreateDatabaseFetcherActor(const TString& database);

} // NKikimr::NMetadata::NModifications
2 changes: 2 additions & 0 deletions ydb/services/metadata/manager/ya.make
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,13 @@ SRCS(
ydb_value_operator.cpp
modification_controller.cpp
object.cpp
fetch_database.cpp
)

PEERDIR(
ydb/library/accessor
ydb/library/actors/core
ydb/library/table_creator
ydb/public/api/protos
ydb/core/protos
ydb/services/bg_tasks/abstract
Expand Down

0 comments on commit 0556e28

Please sign in to comment.