Skip to content

Commit

Permalink
Partition at decimal keys (ydb-platform#10696)
Browse files Browse the repository at this point in the history
  • Loading branch information
@azevaykin
azevaykin committed Dec 6, 2024
1 parent 6a1763d commit 5337648
Show file tree
Hide file tree
Showing 8 changed files with 245 additions and 99 deletions.
60 changes: 43 additions & 17 deletions ydb/core/kqp/ut/query/kqp_params_ut.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -609,6 +609,7 @@ Y_UNIT_TEST_SUITE(KqpParams) {
.AddParam("$ParamDouble").Double(40.5).Build()
.AddParam("$ParamDecimal").Decimal(TDecimalValue("50.5", 22, 9)).Build()
.AddParam("$ParamDecimal35").Decimal(TDecimalValue("655555555555555.5", 35, 10)).Build()
.AddParam("$ParamDecimal0").Decimal(TDecimalValue("9", 1, 0)).Build()
.AddParam("$ParamDyNumber").DyNumber("60.5").Build()
.AddParam("$ParamString").String("StringValue").Build()
.AddParam("$ParamUtf8").Utf8("Utf8Value").Build()
Expand Down Expand Up @@ -670,6 +671,7 @@ Y_UNIT_TEST_SUITE(KqpParams) {
DECLARE $ParamDouble AS Double;
DECLARE $ParamDecimal AS Decimal(22, 9);
DECLARE $ParamDecimal35 AS Decimal(35, 10);
DECLARE $ParamDecimal0 AS Decimal(1, 0);
DECLARE $ParamDyNumber AS DyNumber;
DECLARE $ParamString AS String;
DECLARE $ParamUtf8 AS Utf8;
Expand Down Expand Up @@ -704,6 +706,7 @@ Y_UNIT_TEST_SUITE(KqpParams) {
$ParamDouble AS ValueDouble,
$ParamDecimal AS ValueDecimal,
$ParamDecimal35 AS ValueDecimal35,
$ParamDecimal0 AS ValueDecimal0,
$ParamDyNumber AS ValueDyNumber,
$ParamString AS ValueString,
$ParamUtf8 AS ValueUtf8,
Expand All @@ -729,13 +732,13 @@ Y_UNIT_TEST_SUITE(KqpParams) {

auto actual = ReformatYson(FormatResultSetYson(result.GetResultSet(0)));
auto expected1 = ReformatYson(R"([[
%true;-5;5u;-8;8u;-10;10u;-20;20u;30.5;40.5;"50.5";"655555555555555.5";".605e2";"StringValue";"Utf8Value";"[{Value=50}]";
%true;-5;5u;-8;8u;-10;10u;-20;20u;30.5;40.5;"50.5";"655555555555555.5";"9";".605e2";"StringValue";"Utf8Value";"[{Value=50}]";
"[{\"Value\":60}]";"[{\"Value\":70}]";18271u;1578755093u;1578863917000000u;3600;"2022-03-14,GMT";
"2022-03-14T00:00:00,GMT";"2022-03-14T00:00:00.123000,GMT";["Opt"];["Tuple0";1];[17u;19u];[];["Paul";-5];
[["Key2";20u];["Key1";10u]]
]])");
auto expected2 = ReformatYson(R"([[
%true;-5;5u;-8;8u;-10;10u;-20;20u;30.5;40.5;"50.5";"655555555555555.5";".605e2";"StringValue";"Utf8Value";"[{Value=50}]";
%true;-5;5u;-8;8u;-10;10u;-20;20u;30.5;40.5;"50.5";"655555555555555.5";"9";".605e2";"StringValue";"Utf8Value";"[{Value=50}]";
"[{\"Value\":60}]";"[{\"Value\":70}]";18271u;1578755093u;1578863917000000u;3600;"2022-03-14,GMT";
"2022-03-14T00:00:00,GMT";"2022-03-14T00:00:00.123000,GMT";["Opt"];["Tuple0";1];[17u;19u];[];["Paul";-5];
[["Key1";10u];["Key2";20u]]
Expand Down Expand Up @@ -793,6 +796,7 @@ Y_UNIT_TEST_SUITE(KqpParams) {
--!syntax_v1
CREATE TABLE Table (
Key Int32,
Value1 Decimal(1,0),
Value22 Decimal(22,9),
Value35 Decimal(35,10),
PRIMARY KEY (Key)
Expand All @@ -814,18 +818,19 @@ Y_UNIT_TEST_SUITE(KqpParams) {
auto execSelectQuery = [&] (const TString& query, const NYdb::TParams& params) -> std::tuple<NYdb::EStatus, TString, TResultSet> {
if (QueryService) {
auto result = queryClient.ExecuteQuery(query, NYdb::NQuery::TTxControl::BeginTx().CommitTx(), params).ExtractValueSync();
return {result.GetStatus(), result.GetIssues().ToString(), result.GetResultSet(0)};
return {result.GetStatus(), result.GetIssues().ToString(), result.GetResultSets().size() ? result.GetResultSet(0) : TResultSet({})};
}
else {
auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), params).ExtractValueSync();
return {result.GetStatus(), result.GetIssues().ToString(), result.GetResultSet(0)};
return {result.GetStatus(), result.GetIssues().ToString(), result.GetResultSets().size() ? result.GetResultSet(0) : TResultSet({})};
}
};

// Good case
{
auto upsertParams = tableClient.GetParamsBuilder()
.AddParam("$key").Int32(1).Build()
.AddParam("$value1").Decimal(TDecimalValue("9", 1, 0)).Build()
.AddParam("$value22").Decimal(TDecimalValue("123.321", 22, 9)).Build()
.AddParam("$value35").Decimal(TDecimalValue("555555555555555.1234567890", 35, 10)).Build()
.Build();
Expand All @@ -834,26 +839,28 @@ Y_UNIT_TEST_SUITE(KqpParams) {
{
auto [status, issues] = execUpsertQuery(Q1_(R"(
DECLARE $key AS Int32;
DECLARE $value1 AS Decimal(1,0);
DECLARE $value22 AS Decimal(22,9);
DECLARE $value35 AS Decimal(35,10);
UPSERT INTO Table (Key, Value22, Value35) VALUES
($key, $value22, $value35);
UPSERT INTO Table (Key, Value1, Value22, Value35) VALUES
($key, $value1, $value22, $value35);
)"), upsertParams);
UNIT_ASSERT_VALUES_EQUAL_C(status, EStatus::SUCCESS, issues);
}
// No upsert parameters is declared
{
auto [status, issues] = execUpsertQuery(Q1_(R"(
UPSERT INTO Table (Key, Value22, Value35) VALUES
($key, $value22, $value35);
UPSERT INTO Table (Key, Value1, Value22, Value35) VALUES
($key, $value1, $value22, $value35);
)"), upsertParams);
UNIT_ASSERT_VALUES_EQUAL_C(status, EStatus::SUCCESS, issues);
}

TString expected = R"([[[1];["123.321"];["555555555555555.123456789"]]])";
TString expected = R"([[[1];["9"];["123.321"];["555555555555555.123456789"]]])";
auto selectParams = tableClient.GetParamsBuilder()
.AddParam("$key").Int32(1).Build()
.AddParam("$value1").Decimal(TDecimalValue("9", 1, 0)).Build()
.AddParam("$value22").Decimal(TDecimalValue("123.321", 22, 9)).Build()
.AddParam("$value35").Decimal(TDecimalValue("555555555555555.1234567890", 35, 10)).Build()
.Build();
Expand All @@ -862,10 +869,11 @@ Y_UNIT_TEST_SUITE(KqpParams) {
{
auto [status, issues, resultSet] = execSelectQuery(Q1_(R"(
DECLARE $key AS Int32;
DECLARE $value1 AS Decimal(1,0);
DECLARE $value22 AS Decimal(22,9);
DECLARE $value35 AS Decimal(35,10);
SELECT * FROM Table WHERE Key = $key AND Value22 = $value22 AND Value35 = $value35;
SELECT * FROM Table WHERE Key = $key AND Value1 = $value1 AND Value22 = $value22 AND Value35 = $value35;
)"), selectParams);
UNIT_ASSERT_VALUES_EQUAL_C(status, EStatus::SUCCESS, issues);
CompareYson(expected, FormatResultSetYson(resultSet));
Expand All @@ -874,13 +882,27 @@ Y_UNIT_TEST_SUITE(KqpParams) {
// No select parameters is declared
{
auto [status, issues, resultSet] = execSelectQuery(Q1_(R"(
SELECT * FROM Table WHERE Key = $key AND Value22 = $value22 AND Value35 = $value35;
SELECT * FROM Table WHERE Key = $key AND Value1 = $value1 AND Value22 = $value22 AND Value35 = $value35;
)"), selectParams);
UNIT_ASSERT_VALUES_EQUAL_C(status, EStatus::SUCCESS, issues);
CompareYson(expected, FormatResultSetYson(resultSet));
}
}

// Declare wrong decimal params
{
auto params = tableClient.GetParamsBuilder()
.AddParam("$value99").Decimal(TDecimalValue("0", 99, 99)).Build()
.Build();

auto [status, issues, _] = execSelectQuery(Q1_(R"(
DECLARE $value99 AS Decimal(99,99);
SELECT $value99 AS value99;
)"), params);
UNIT_ASSERT_VALUES_EQUAL(status, EStatus::GENERIC_ERROR);
UNIT_ASSERT_STRING_CONTAINS(issues, "Invalid decimal precision: 99");
}

// Declare decimal params mismatch
{
auto upsertParams = tableClient.GetParamsBuilder()
Expand Down Expand Up @@ -940,35 +962,39 @@ Y_UNIT_TEST_SUITE(KqpParams) {
{
auto upsertParams = tableClient.GetParamsBuilder()
.AddParam("$key").Int32(1001).Build()
.AddParam("$value1").Decimal(TDecimalValue("10", 1, 0)).Build()
.AddParam("$value22").Decimal(TDecimalValue("12345678901234567890.1234567891", 22, 9)).Build()
.AddParam("$value35").Decimal(TDecimalValue("1234567890123456789012345678901234567890.1234567891", 35, 10)).Build()
.Build();

auto [status, issues] = execUpsertQuery(Q1_(R"(
DECLARE $key AS Int32;
DECLARE $value1 AS Decimal(1,0);
DECLARE $value22 AS Decimal(22,9);
DECLARE $value35 AS Decimal(35,10);
UPSERT INTO Table (Key, Value22, Value35) VALUES
($key, $value22, $value35);
UPSERT INTO Table (Key, Value1, Value22, Value35) VALUES
($key, $value1, $value22, $value35);
)"), upsertParams);
UNIT_ASSERT_VALUES_EQUAL_C(status, EStatus::SUCCESS, issues);
}
// Good case: Upsert inf Decimal
{
auto upsertParams = tableClient.GetParamsBuilder()
.AddParam("$key").Int32(1002).Build()
.AddParam("$value1").Decimal(TDecimalValue("inf", 1, 0)).Build()
.AddParam("$value22").Decimal(TDecimalValue("inf", 22, 9)).Build()
.AddParam("$value35").Decimal(TDecimalValue("inf", 35, 10)).Build()
.Build();

auto [status, issues] = execUpsertQuery(Q1_(R"(
DECLARE $key AS Int32;
DECLARE $value1 AS Decimal(1,0);
DECLARE $value22 AS Decimal(22,9);
DECLARE $value35 AS Decimal(35,10);
UPSERT INTO Table (Key, Value22, Value35) VALUES
($key, $value22, $value35);
UPSERT INTO Table (Key, Value1, Value22, Value35) VALUES
($key, $value1, $value22, $value35);
)"), upsertParams);
UNIT_ASSERT_VALUES_EQUAL_C(status, EStatus::SUCCESS, issues);
}
Expand All @@ -980,8 +1006,8 @@ Y_UNIT_TEST_SUITE(KqpParams) {
)"), emptyParams);
UNIT_ASSERT_VALUES_EQUAL_C(status, EStatus::SUCCESS, issues);
TString expected = R"([
[[1001];["inf"];["inf"]];
[[1002];["inf"];["inf"]]
[[1001];["inf"];["inf"];["inf"]];
[[1002];["inf"];["inf"];["inf"]]
])";
TString actual = FormatResultSetYson(resultSet);
CompareYson(expected, actual);
Expand Down
4 changes: 2 additions & 2 deletions ydb/core/kqp/ut/scan/kqp_scan_ut.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,9 +254,9 @@ Y_UNIT_TEST_SUITE(KqpScan) {
const NYdb::NTable::TTableDescription& tableDescription = describeResult.GetTableDescription();
const TVector<NYdb::NTable::TKeyRange>& keyRanges = tableDescription.GetKeyRanges();
const TVector<NYdb::NTable::TTableColumn>& columns = tableDescription.GetTableColumns();
UNIT_ASSERT_VALUES_EQUAL(columns.size(), 2);
UNIT_ASSERT_VALUES_EQUAL(columns.size(), 4);
UNIT_ASSERT_STRINGS_EQUAL(columns[0].Type.ToString(), "Decimal(22,9)?");
UNIT_ASSERT_STRINGS_EQUAL(columns[1].Type.ToString(), "Decimal(22,9)?");
UNIT_ASSERT_STRINGS_EQUAL(columns[1].Type.ToString(), "Decimal(35,10)?");
auto extractValue = [](const TValue& val) {
auto parser = TValueParser(val);
parser.OpenTuple();
Expand Down
43 changes: 35 additions & 8 deletions ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2294,6 +2294,16 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
);)", tableName.c_str(), precision, scale);
auto createResult = session.ExecuteSchemeQuery(createQuery).GetValueSync();

if (precision == 0) {
UNIT_ASSERT_VALUES_EQUAL_C(createResult.GetStatus(), EStatus::GENERIC_ERROR, createResult.GetIssues().ToString());
UNIT_ASSERT_STRING_CONTAINS(createResult.GetIssues().ToString(), "Invalid decimal precision");
return;
}
if (precision == 33) {
UNIT_ASSERT_VALUES_EQUAL_C(createResult.GetStatus(), EStatus::GENERIC_ERROR, createResult.GetIssues().ToString());
UNIT_ASSERT_STRING_CONTAINS(createResult.GetIssues().ToString(), "Invalid decimal parameters");
return;
}
if (precision == 36) {
UNIT_ASSERT_VALUES_EQUAL_C(createResult.GetStatus(), EStatus::GENERIC_ERROR, createResult.GetIssues().ToString());
UNIT_ASSERT_STRING_CONTAINS(createResult.GetIssues().ToString(), "Invalid decimal precision");
Expand Down Expand Up @@ -2324,10 +2334,13 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
UNIT_ASSERT_VALUES_EQUAL(decimalType.Scale, scale);
};

createAndCheck(0, 0);
createAndCheck(1, 0);
createAndCheck(2, 1);
createAndCheck(22, 9);
createAndCheck(35, 9);
createAndCheck(35, 10);
createAndCheck(22, 20);
createAndCheck(33, 34);
createAndCheck(36, 35);
createAndCheck(999, 99);
}
Expand Down Expand Up @@ -2657,6 +2670,16 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
)", tableName.c_str(), columnName.c_str(), precision, scale);
auto result = session.ExecuteSchemeQuery(query).GetValueSync();

if (precision == 0) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString());
UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "Invalid decimal precision");
return;
}
if (precision == 33) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString());
UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "Invalid decimal parameters");
return;
}
if (precision == 36) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString());
UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "Invalid decimal precision");
Expand All @@ -2671,18 +2694,21 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
};

addColumn(0, 0);
addColumn(1, 0);
addColumn(2, 1);
addColumn(22, 9);
addColumn(35, 9);
addColumn(35, 10);
addColumn(22, 20);
addColumn(33, 34);
addColumn(36, 35);
addColumn(999, 99);

TDescribeTableResult describe = session.DescribeTable(tableName).GetValueSync();
UNIT_ASSERT_EQUAL_C(describe.GetStatus(), EStatus::SUCCESS, describe.GetIssues().ToString());
auto tableDesc = describe.GetTableDescription();
TVector<TTableColumn> columns = tableDesc.GetTableColumns();
UNIT_ASSERT_VALUES_EQUAL(columns.size(), 6);
UNIT_ASSERT_VALUES_EQUAL(columns.size(), 7);

auto checkColumn = [&] (ui64 columnIdx, ui32 precision, ui32 scale) {
TType valueType = columns[columnIdx].Type;
Expand All @@ -2691,16 +2717,17 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
UNIT_ASSERT_EQUAL(optionalKind, TTypeParser::ETypeKind::Optional);
parser.OpenOptional();
auto kind = parser.GetKind();
UNIT_ASSERT_EQUAL(kind, TTypeParser::ETypeKind::Decimal);
UNIT_ASSERT_VALUES_EQUAL(kind, TTypeParser::ETypeKind::Decimal);
TDecimalType decimalType = parser.GetDecimal();
UNIT_ASSERT_VALUES_EQUAL(decimalType.Precision, precision);
UNIT_ASSERT_VALUES_EQUAL(decimalType.Scale, scale);
};

checkColumn(2, 2, 1);
checkColumn(3, 22, 9);
checkColumn(4, 35,9);
checkColumn(5, 22, 20);
checkColumn(0,22, 20);
checkColumn(3, 1, 0);
checkColumn(4, 2, 1);
checkColumn(5, 22,9);
checkColumn(6, 35, 10);
}

Y_UNIT_TEST(CreateUserWithPassword) {
Expand Down
16 changes: 16 additions & 0 deletions ydb/core/scheme_types/scheme_decimal_type.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,18 @@

namespace NKikimr::NScheme {

TDecimalType::TDecimalType(ui32 precision, ui32 scale)
: Precision(precision)
, Scale(scale)
{
TString error;
Y_ABORT_UNLESS(Validate(precision, scale, error), "%s", error.c_str());
}

bool TDecimalType::operator==(const TDecimalType& other) const {
return Precision == other.Precision && Scale == other.Scale;
}

TString TDecimalType::CellValueToString(const std::pair<ui64, i64>& cellValue) const {
return NYql::NDecimal::ToString(NYql::NDecimal::FromHalfs(cellValue.first, cellValue.second),
Precision, Scale);
Expand Down Expand Up @@ -44,6 +56,10 @@ const std::optional<TDecimalType> TDecimalType::ParseTypeName(const TStringBuf&
}

bool TDecimalType::Validate(ui32 precision, ui32 scale, TString& error) {
if (precision == 0) {
error = Sprintf("Decimal precision should not be zero");
return false;
}
if (precision > NKikimr::NScheme::DECIMAL_MAX_PRECISION) {
error = Sprintf("Decimal precision %u should be less than %u", precision, NKikimr::NScheme::DECIMAL_MAX_PRECISION);
return false;
Expand Down
14 changes: 3 additions & 11 deletions ydb/core/scheme_types/scheme_decimal_type.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,25 +11,17 @@ namespace NKikimr::NScheme {

class TDecimalType {
public:
constexpr TDecimalType(ui32 precision, ui32 scale)
: Precision(precision)
, Scale(scale)
{
Y_ABORT_UNLESS(Precision);
Y_ABORT_UNLESS(Scale);
}
TDecimalType(ui32 precision, ui32 scale);

constexpr bool operator==(const TDecimalType& other) const {
return Precision == other.Precision && Scale == other.Scale;
}
bool operator==(const TDecimalType& other) const;

TString CellValueToString(const std::pair<ui64, i64>& cellValue) const;
void CellValueToStream(const std::pair<ui64, i64>& cellValue, IOutputStream& out) const;

static const std::optional<TDecimalType> ParseTypeName(const TStringBuf& typeName);
static bool Validate(ui32 precision, ui32 scale, TString& error);

constexpr static TDecimalType Default() {
static TDecimalType Default() {
return TDecimalType(DECIMAL_PRECISION, DECIMAL_SCALE);
}
private:
Expand Down
2 changes: 1 addition & 1 deletion ydb/public/sdk/cpp/client/ydb_value/value.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ class TTypeBuilder : public TMoveOnly {
struct TDecimalValue {
TString ToString() const;
TDecimalValue(const Ydb::Value& decimalValueProto, const TDecimalType& decimalType);
TDecimalValue(const TString& decimalString, ui8 precision = 22, ui8 scale = 9);
TDecimalValue(const TString& decimalString, ui8 precision, ui8 scale);

TDecimalType DecimalType_;
ui64 Low_;
Expand Down
Loading

0 comments on commit 5337648

Please sign in to comment.