Skip to content

Commit

Permalink
Add default policy to ColumnMeta (ydb-platform#1791)
Browse files Browse the repository at this point in the history
  • Loading branch information
@shnikd @gridnevvvit
shnikd authored and gridnevvvit committed Feb 21, 2024
1 parent 0fd6659 commit 2d35372
Show file tree
Hide file tree
Showing 4 changed files with 275 additions and 4 deletions.
66 changes: 64 additions & 2 deletions ydb/core/tx/schemeshard/ut_export/ut_export.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -281,8 +281,42 @@ Y_UNIT_TEST_SUITE(TExportToS3Tests) {

const TVector<TString> tables = {R"(
Name: "Table"
Columns { Name: "key" Type: "Utf8" }
Columns { Name: "value" Type: "Utf8" }
Columns {
Name: "key"
Type: "Utf8"
DefaultFromLiteral {
type {
optional_type {
item {
type_id: UTF8
}
}
}
value {
items {
text_value: "b"
}
}
}
}
Columns {
Name: "value"
Type: "Utf8"
DefaultFromLiteral {
type {
optional_type {
item {
type_id: UTF8
}
}
}
value {
items {
text_value: "a"
}
}
}
}
KeyColumnNames: ["key"]
PartitionConfig {
ColumnFamilies {
Expand Down Expand Up @@ -331,6 +365,20 @@ Y_UNIT_TEST_SUITE(TExportToS3Tests) {
}
}
not_null: false
from_literal {
type {
optional_type {
item {
type_id: UTF8
}
}
}
value {
items {
text_value: "b"
}
}
}
}
columns {
name: "value"
Expand All @@ -342,6 +390,20 @@ columns {
}
}
not_null: false
from_literal {
type {
optional_type {
item {
type_id: UTF8
}
}
}
value {
items {
text_value: "a"
}
}
}
}
primary_key: "key"
storage_settings {
Expand Down
148 changes: 148 additions & 0 deletions ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,79 @@ Y_UNIT_TEST_SUITE(TRestoreTests) {
NKqp::CompareYson(data.YsonStr, content);
}

bool CheckDefaultFromLiteral(const NKikimrSchemeOp::TTableDescription& desc) {
for (const auto& column: desc.GetColumns()) {
if (column.GetName() == "value") {
switch (column.GetDefaultValueCase()) {
case NKikimrSchemeOp::TColumnDescription::kDefaultFromLiteral: {
const auto& fromLiteral = column.GetDefaultFromLiteral();

TString str;
google::protobuf::TextFormat::PrintToString(fromLiteral, &str);

TString result = R"(type {
optional_type {
item {
type_id: UTF8
}
}
}
value {
items {
text_value: "value1"
}
}
)";
return str == result;
}
default: break;
}
break;
}
}
return false;
}

Y_UNIT_TEST_WITH_COMPRESSION(ShouldSucceedWithDefaultFromLiteral) {
TTestBasicRuntime runtime;

const auto data = GenerateTestData(Codec, "a", 1);

Restore(runtime, R"(
Name: "Table"
Columns { Name: "key" Type: "Utf8" }
Columns {
Name: "value"
Type: "Utf8"
DefaultFromLiteral {
type {
optional_type {
item {
type_id: UTF8
}
}
}
value {
items {
text_value: "value1"
}
}
}
}
KeyColumnNames: ["key"]
)", {data});

auto content = ReadTable(runtime, TTestTxConfig::FakeHiveTablets);
NKqp::CompareYson(data.YsonStr, content);

const auto desc = DescribePath(runtime, "/MyRoot/Table", true, true);
UNIT_ASSERT_VALUES_EQUAL(desc.GetStatus(), NKikimrScheme::StatusSuccess);

const auto& table = desc.GetPathDescription().GetTable();

UNIT_ASSERT_C(CheckDefaultFromLiteral(table), "Invalid default value");
}

Y_UNIT_TEST_WITH_COMPRESSION(ShouldSucceedOnMultiShardTable) {
TTestBasicRuntime runtime;

Expand Down Expand Up @@ -729,6 +802,81 @@ Y_UNIT_TEST_SUITE(TRestoreTests) {
TestGetImport(runtime, txId, "/MyRoot");
}

Y_UNIT_TEST(ShouldRestoreDefaultValuesFromLiteral) {
TPortManager portManager;
const ui16 port = portManager.GetPort();

TS3Mock s3Mock({}, TS3Mock::TSettings(port));
UNIT_ASSERT(s3Mock.Start());

TTestBasicRuntime runtime;
TTestEnv env(runtime);
ui64 txId = 100;

runtime.SetLogPriority(NKikimrServices::DATASHARD_BACKUP, NActors::NLog::PRI_TRACE);
runtime.SetLogPriority(NKikimrServices::DATASHARD_RESTORE, NActors::NLog::PRI_TRACE);
runtime.SetLogPriority(NKikimrServices::EXPORT, NActors::NLog::PRI_TRACE);
runtime.SetLogPriority(NKikimrServices::IMPORT, NActors::NLog::PRI_TRACE);

TestCreateTable(runtime, ++txId, "/MyRoot", R"(
Name: "Original"
Columns { Name: "key" Type: "Utf8" }
Columns {
Name: "value"
Type: "Utf8"
DefaultFromLiteral {
type {
optional_type {
item {
type_id: UTF8
}
}
}
value {
items {
text_value: "value1"
}
}
}
}
KeyColumnNames: ["key"]
)");
env.TestWaitNotification(runtime, txId);

TestExport(runtime, ++txId, "/MyRoot", Sprintf(R"(
ExportToS3Settings {
endpoint: "localhost:%d"
scheme: HTTP
items {
source_path: "/MyRoot/Original"
destination_prefix: ""
}
}
)", port));
env.TestWaitNotification(runtime, txId);
TestGetExport(runtime, txId, "/MyRoot");

TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"(
ImportFromS3Settings {
endpoint: "localhost:%d"
scheme: HTTP
items {
source_prefix: ""
destination_path: "/MyRoot/Restored"
}
}
)", port));
env.TestWaitNotification(runtime, txId);
TestGetImport(runtime, txId, "/MyRoot");

const auto desc = DescribePath(runtime, "/MyRoot/Restored", true, true);
UNIT_ASSERT_VALUES_EQUAL(desc.GetStatus(), NKikimrScheme::StatusSuccess);

const auto& table = desc.GetPathDescription().GetTable();

UNIT_ASSERT_C(CheckDefaultFromLiteral(table), "Invalid default value");
}

Y_UNIT_TEST(ExportImportPg) {
TTestBasicRuntime runtime;
TTestEnv env(runtime, TTestEnvOptions().EnableTablePgTypes(true));
Expand Down
61 changes: 59 additions & 2 deletions ydb/core/ydb_convert/table_description.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,11 +151,11 @@ bool BuildAlterTableAddIndexRequest(const Ydb::Table::AlterTableRequest* req, NK
if (flags & NKqpProto::TKqpSchemeOperation::FLAG_PG_MODE) {
settings->set_pg_mode(true);
}

if (flags & NKqpProto::TKqpSchemeOperation::FLAG_IF_NOT_EXISTS) {
settings->set_if_not_exist(true);
}

settings->set_source_path(req->path());
auto tableIndex = settings->mutable_index();
tableIndex->CopyFrom(req->add_indexes(0));
Expand Down Expand Up @@ -390,6 +390,54 @@ static Ydb::Type* AddColumn(Ydb::Table::ColumnMeta* newColumn, const TColumn& co
}
}
newColumn->set_not_null(column.GetNotNull());
return columnType;
}

template <>
Ydb::Type* AddColumn<NKikimrSchemeOp::TColumnDescription>(Ydb::Table::ColumnMeta* newColumn, const NKikimrSchemeOp::TColumnDescription& column) {
newColumn->set_name(column.GetName());

Ydb::Type* columnType = nullptr;
auto* typeDesc = NPg::TypeDescFromPgTypeName(column.GetType());
if (typeDesc) {
columnType = newColumn->mutable_type();
auto* pg = columnType->mutable_pg_type();
pg->set_type_name(NPg::PgTypeNameFromTypeDesc(typeDesc));
pg->set_type_modifier(NPg::TypeModFromPgTypeName(column.GetType()));
pg->set_oid(NPg::PgTypeIdFromTypeDesc(typeDesc));
pg->set_typlen(0);
pg->set_typmod(0);
} else {
NYql::NProto::TypeIds protoType;
if (!NYql::NProto::TypeIds_Parse(column.GetType(), &protoType)) {
throw NYql::TErrorException(NKikimrIssues::TIssuesIds::DEFAULT_ERROR)
<< "Got invalid type: " << column.GetType() << " for column: " << column.GetName();
}

if (column.GetNotNull()) {
columnType = newColumn->mutable_type();
} else {
columnType = newColumn->mutable_type()->mutable_optional_type()->mutable_item();
}
Y_ENSURE(columnType);
if (protoType == NYql::NProto::TypeIds::Decimal) {
auto typeParams = columnType->mutable_decimal_type();
// TODO: Change TEvDescribeSchemeResult to return decimal params
typeParams->set_precision(22);
typeParams->set_scale(9);
} else {
NMiniKQL::ExportPrimitiveTypeToProto(protoType, *columnType);
}
}
newColumn->set_not_null(column.GetNotNull());
switch (column.GetDefaultValueCase()) {
case NKikimrSchemeOp::TColumnDescription::kDefaultFromLiteral: {
auto fromLiteral = newColumn->mutable_from_literal();
*fromLiteral = column.GetDefaultFromLiteral();
break;
}
default: break;
}

return columnType;
}
Expand Down Expand Up @@ -607,6 +655,15 @@ bool FillColumnDescription(NKikimrSchemeOp::TTableDescription& out,
if (!column.family().empty()) {
cd->SetFamilyName(column.family());
}

switch (column.default_value_case()) {
case Ydb::Table::ColumnMeta::kFromLiteral: {
auto fromLiteral = cd->MutableDefaultFromLiteral();
*fromLiteral = column.from_literal();
break;
}
default: break;
}
}

return true;
Expand Down
4 changes: 4 additions & 0 deletions ydb/public/api/protos/ydb_table.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -342,6 +342,10 @@ message ColumnMeta {
string family = 3;
// Column nullability
optional bool not_null = 4;
// Column default value option
oneof default_value {
TypedValue from_literal = 5;
}
}

message DateTypeColumnModeSettings {
Expand Down

0 comments on commit 2d35372

Please sign in to comment.