diff --git a/ydb/core/tx/datashard/datashard__conditional_erase_rows.cpp b/ydb/core/tx/datashard/datashard__conditional_erase_rows.cpp index a84bf4d2f565..3596ee578c13 100644 --- a/ydb/core/tx/datashard/datashard__conditional_erase_rows.cpp +++ b/ydb/core/tx/datashard/datashard__conditional_erase_rows.cpp @@ -470,6 +470,9 @@ static bool CheckUnit(NScheme::TTypeInfo type, NKikimrSchemeOp::TTTLSettings::EU case NScheme::NTypeIds::Date: case NScheme::NTypeIds::Datetime: case NScheme::NTypeIds::Timestamp: + case NScheme::NTypeIds::Date32: + case NScheme::NTypeIds::Datetime64: + case NScheme::NTypeIds::Timestamp64: if (unit == NKikimrSchemeOp::TTTLSettings::UNIT_AUTO) { return true; } else { diff --git a/ydb/core/tx/datashard/datashard_ut_erase_rows.cpp b/ydb/core/tx/datashard/datashard_ut_erase_rows.cpp index 5a65a67765ee..73ffc5988bbe 100644 --- a/ydb/core/tx/datashard/datashard_ut_erase_rows.cpp +++ b/ydb/core/tx/datashard/datashard_ut_erase_rows.cpp @@ -428,14 +428,18 @@ Y_UNIT_TEST_SUITE(EraseRowsTests) { TProto::TEvEraseResponse::SCHEME_ERROR, "Cell count doesn't match row scheme"); } - void ConditionalEraseShouldSuccess(const TString& ttlColType, EUnit unit, const TString& toUpload, const TString& afterErase) { + void ConditionalEraseShouldSuccess(const TString& ttlColType, EUnit unit, const TString& toUpload, const TString& afterErase, const bool enableDatetime64 = false) { using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; + NKikimrConfig::TFeatureFlags featureFlags; + featureFlags.SetEnableTableDatetime64(enableDatetime64); + TPortManager pm; TServerSettings serverSettings(pm.GetPort(2134)); serverSettings .SetDomainName("Root") - .SetUseRealThreads(false); + .SetUseRealThreads(false) + .SetFeatureFlags(featureFlags); TServer::TPtr server = new TServer(serverSettings); auto& runtime = *server->GetRuntime(); @@ -615,6 +619,50 @@ key = 4, value = (empty maybe) )"); } + Y_UNIT_TEST(ConditionalEraseRowsShouldEraseOnDate32) { + ConditionalEraseShouldSuccess("Date32", TUnit::AUTO, R"( +UPSERT INTO `/Root/table-1` (key, value) VALUES +(1, CAST("1960-01-01" AS Date32)), +(2, CAST("1970-01-01" AS Date32)), +(3, CAST("1990-03-01" AS Date32)), +(4, CAST("2030-04-15" AS Date32)), +(5, NULL); + )", R"( +key = 4, value = 22019 +key = 5, value = (empty maybe) + )", true); + } + + Y_UNIT_TEST(ConditionalEraseRowsShouldEraseOnDatetime64) { + ConditionalEraseShouldSuccess("Datetime64", TUnit::AUTO, R"( +UPSERT INTO `/Root/table-1` (key, value) VALUES +(1, CAST("1960-01-01T00:00:00Z" AS Datetime64)), +(2, CAST("1970-01-01T00:00:00Z" AS Datetime64)), +(3, CAST("1990-03-01T00:00:00Z" AS Datetime64)), +(4, CAST("2030-04-15T00:00:00Z" AS Datetime64)), +(5, NULL); + )", R"( +key = 4, value = 1902441600 +key = 5, value = (empty maybe) + )", true); + } + + Y_UNIT_TEST(ConditionalEraseRowsShouldEraseOnTimestamp64) { + ConditionalEraseShouldSuccess("Timestamp64", TUnit::AUTO, R"( +UPSERT INTO `/Root/table-1` (key, value) VALUES +(1, CAST("1960-01-01T00:00:00.000000Z" AS Timestamp64)), +(2, CAST("1970-01-01T00:00:00.000000Z" AS Timestamp64)), +(3, CAST("1990-03-01T00:00:00.000000Z" AS Timestamp64)), +(4, CAST("2030-04-15T00:00:00.000000Z" AS Timestamp64)), +(5, NULL); + )", R"( +key = 4, value = 1902441600000000 +key = 5, value = (empty maybe) + )", true); + } + + + Y_UNIT_TEST(ConditionalEraseRowsShouldFailOnVariousErrors) { using TEvResponse = TEvDataShard::TEvConditionalEraseRowsResponse; diff --git a/ydb/core/tx/datashard/erase_rows_condition.cpp b/ydb/core/tx/datashard/erase_rows_condition.cpp index 4d895e02ff81..98c4b7d83fa7 100644 --- a/ydb/core/tx/datashard/erase_rows_condition.cpp +++ b/ydb/core/tx/datashard/erase_rows_condition.cpp @@ -56,7 +56,7 @@ class TExpirationCondition: public IEraseRowsCondition { return WallClockDyNumber; } - bool Check(ui64 value) const { + bool CheckUi64(ui64 value) const { switch (Type) { // 'date-type column' mode case NScheme::NTypeIds::Date: @@ -87,7 +87,26 @@ class TExpirationCondition: public IEraseRowsCondition { } } - bool Check(TStringBuf value) const { + bool CheckI64(i64 value) const { + + // Dates before 1970 are deleted by TTL + if (value < 0) + return true; + + switch (Type) { + // 'big date-type column' mode + case NScheme::NTypeIds::Date32: + return TInstant::Days(value) <= WallClockInstant; + case NScheme::NTypeIds::Datetime64: + return TInstant::Seconds(value) <= WallClockInstant; + case NScheme::NTypeIds::Timestamp64: + return TInstant::MicroSeconds(value) <= WallClockInstant; + default: + Y_ABORT("Unreachable"); + } + } + + bool CheckStr(TStringBuf value) const { switch (Type) { // 'value since epoch' mode case NScheme::NTypeIds::DyNumber: @@ -147,15 +166,20 @@ class TExpirationCondition: public IEraseRowsCondition { switch (Type) { case NScheme::NTypeIds::Date: - return Check(cell.AsValue()); + return CheckUi64(cell.AsValue()); case NScheme::NTypeIds::Datetime: case NScheme::NTypeIds::Uint32: - return Check(cell.AsValue()); + return CheckUi64(cell.AsValue()); case NScheme::NTypeIds::Timestamp: case NScheme::NTypeIds::Uint64: - return Check(cell.AsValue()); + return CheckUi64(cell.AsValue()); + case NScheme::NTypeIds::Date32: + return CheckI64(cell.AsValue()); + case NScheme::NTypeIds::Datetime64: + case NScheme::NTypeIds::Timestamp64: + return CheckI64(cell.AsValue()); case NScheme::NTypeIds::DyNumber: - return Check(cell.AsBuf()); + return CheckStr(cell.AsBuf()); default: return false; } diff --git a/ydb/core/tx/datashard/read_table_scan.cpp b/ydb/core/tx/datashard/read_table_scan.cpp index bc0be6f68684..8b21023527fc 100644 --- a/ydb/core/tx/datashard/read_table_scan.cpp +++ b/ydb/core/tx/datashard/read_table_scan.cpp @@ -156,6 +156,22 @@ Y_FORCE_INLINE bool AddCell(TOutValue& row, NScheme::TTypeInfo type, const TCell val.set_int64_value(value); break; } + case NUdf::TDataType::Id: { + i32 value; + if (!cell.ToValue(value, err)) + return false; + val.set_int32_value(value); + break; + } + case NUdf::TDataType::Id: + case NUdf::TDataType::Id: + case NUdf::TDataType::Id: { + i64 value; + if (!cell.ToValue(value, err)) + return false; + val.set_int64_value(value); + break; + } case NUdf::TDataType::Id: { const auto json = NBinaryJson::SerializeToJson(TStringBuf(cell.Data(), cell.Size())); val.set_text_value(json); diff --git a/ydb/core/tx/datashard/ut_common/datashard_ut_common.cpp b/ydb/core/tx/datashard/ut_common/datashard_ut_common.cpp index 17bd59bda86b..1421e3afa670 100644 --- a/ydb/core/tx/datashard/ut_common/datashard_ut_common.cpp +++ b/ydb/core/tx/datashard/ut_common/datashard_ut_common.cpp @@ -2357,6 +2357,9 @@ namespace { PRINT_PRIMITIVE(Date); PRINT_PRIMITIVE(Datetime); PRINT_PRIMITIVE(Timestamp); + PRINT_PRIMITIVE(Date32); + PRINT_PRIMITIVE(Datetime64); + PRINT_PRIMITIVE(Timestamp64); PRINT_PRIMITIVE(String); PRINT_PRIMITIVE(DyNumber); diff --git a/ydb/core/tx/schemeshard/schemeshard_validate_ttl.cpp b/ydb/core/tx/schemeshard/schemeshard_validate_ttl.cpp index cc447f07015b..997001bafdc7 100644 --- a/ydb/core/tx/schemeshard/schemeshard_validate_ttl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_validate_ttl.cpp @@ -59,6 +59,12 @@ bool ValidateTtlSettings(const NKikimrSchemeOp::TTTLSettings& ttl, return false; } + const TInstant now = TInstant::Now(); + if (enabled.GetExpireAfterSeconds() > now.Seconds()) { + errStr = Sprintf("TTL should be less than %" PRIu64 " seconds (%" PRIu64 " days, %" PRIu64 " years). The ttl behaviour is undefined before 1970.", now.Seconds(), now.Days(), now.Days() / 365); + return false; + } + if (enabled.HasSysSettings()) { const auto& sys = enabled.GetSysSettings(); if (TDuration::FromValue(sys.GetRunInterval()) < subDomain.GetTtlMinRunInterval()) { diff --git a/ydb/core/tx/schemeshard/ut_ttl/ut_ttl.cpp b/ydb/core/tx/schemeshard/ut_ttl/ut_ttl.cpp index 20b7fcde9824..ce57f14992b3 100644 --- a/ydb/core/tx/schemeshard/ut_ttl/ut_ttl.cpp +++ b/ydb/core/tx/schemeshard/ut_ttl/ut_ttl.cpp @@ -155,6 +155,29 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) { )", {NKikimrScheme::StatusSchemeError}); } + Y_UNIT_TEST(CreateTableShouldFailOnBeforeEpochTTL) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + // An attempt to create 100-year TTL. + // The TTL behaviour is undefined before 1970, + // so it's forbidden. + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "TTLEnabledTable" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "modified_at" Type: "Timestamp" } + KeyColumnNames: ["key"] + TTLSettings { + Enabled { + ColumnName: "modified_at" + ExpireAfterSeconds: 3153600000 + } + } + )", {NKikimrScheme::StatusSchemeError}); + } + void CreateTableOnIndexedTable(NKikimrSchemeOp::EIndexType indexType) { TTestBasicRuntime runtime; TTestEnv env(runtime); diff --git a/ydb/core/ydb_convert/ydb_convert.cpp b/ydb/core/ydb_convert/ydb_convert.cpp index 54822c667232..2d92bc3a4421 100644 --- a/ydb/core/ydb_convert/ydb_convert.cpp +++ b/ydb/core/ydb_convert/ydb_convert.cpp @@ -307,6 +307,14 @@ Y_FORCE_INLINE void ConvertData(NUdf::TDataTypeId typeId, const NKikimrMiniKQL:: case NUdf::TDataType::Id: res.set_int64_value(value.GetInt64()); break; + case NUdf::TDataType::Id: + res.set_int32_value(value.GetInt32()); + break; + case NUdf::TDataType::Id: + case NUdf::TDataType::Id: + case NUdf::TDataType::Id: + res.set_int64_value(value.GetInt64()); + break; case NUdf::TDataType::Id: case NUdf::TDataType::Id: { res.set_low_128(value.GetLow128()); @@ -446,6 +454,34 @@ Y_FORCE_INLINE void ConvertData(NUdf::TDataTypeId typeId, const Ydb::Value& valu } res.SetInt64(value.int64_value()); break; + case NUdf::TDataType::Id: + CheckTypeId(value.value_case(), Ydb::Value::kInt32Value, "Date"); + if (value.int32_value() >= NUdf::MAX_DATE32) { + throw yexception() << "Invalid Date32 value"; + } + res.SetInt32(value.int32_value()); + break; + case NUdf::TDataType::Id: + CheckTypeId(value.value_case(), Ydb::Value::kInt64Value, "Datetime"); + if (value.int64_value() >= NUdf::MAX_DATETIME64) { + throw yexception() << "Invalid Datetime64 value"; + } + res.SetInt64(value.int64_value()); + break; + case NUdf::TDataType::Id: + CheckTypeId(value.value_case(), Ydb::Value::kInt64Value, "Timestamp"); + if (value.int64_value() >= NUdf::MAX_TIMESTAMP64) { + throw yexception() << "Invalid Timestamp64 value"; + } + res.SetInt64(value.int64_value()); + break; + case NUdf::TDataType::Id: + CheckTypeId(value.value_case(), Ydb::Value::kInt64Value, "Interval"); + if (std::abs(value.int64_value()) >= NUdf::MAX_INTERVAL64) { + throw yexception() << "Invalid Interval64 value"; + } + res.SetInt64(value.int64_value()); + break; case NUdf::TDataType::Id: CheckTypeId(value.value_case(), Ydb::Value::kLow128, "Uuid"); res.SetLow128(value.low_128()); @@ -1075,6 +1111,22 @@ bool CheckValueData(NScheme::TTypeInfo type, const TCell& cell, TString& err) { ok = (ui64)std::abs(cell.AsValue()) < NUdf::MAX_TIMESTAMP; break; + case NScheme::NTypeIds::Date32: + ok = cell.AsValue() < NUdf::MAX_DATE32; + break; + + case NScheme::NTypeIds::Datetime64: + ok = cell.AsValue() < NUdf::MAX_DATETIME64; + break; + + case NScheme::NTypeIds::Timestamp64: + ok = cell.AsValue() < NUdf::MAX_TIMESTAMP64; + break; + + case NScheme::NTypeIds::Interval64: + ok = std::abs(cell.AsValue()) < NUdf::MAX_INTERVAL64; + break; + case NScheme::NTypeIds::Utf8: ok = NYql::IsUtf8(cell.AsBuf()); break; @@ -1308,6 +1360,18 @@ void ProtoValueFromCell(NYdb::TValueBuilder& vb, const NScheme::TTypeInfo& typeI case EPrimitiveType::Interval: vb.Interval(cell.AsValue()); break; + case EPrimitiveType::Date32: + vb.Date32(cell.AsValue()); + break; + case EPrimitiveType::Datetime64: + vb.Datetime64(cell.AsValue()); + break; + case EPrimitiveType::Timestamp64: + vb.Timestamp64(cell.AsValue()); + break; + case EPrimitiveType::Interval64: + vb.Interval64(cell.AsValue()); + break; case EPrimitiveType::TzDate: vb.TzDate(getString()); break;