Skip to content

Commit

Permalink
Merge 0ca7ce6 into c69bdac
Browse files Browse the repository at this point in the history
  • Loading branch information
@ivanmorozov333
ivanmorozov333 authored Dec 23, 2024
2 parents c69bdac + 0ca7ce6 commit 4623dc8
Show file tree
Hide file tree
Showing 16 changed files with 736 additions and 260 deletions.
109 changes: 83 additions & 26 deletions ydb/core/kqp/ut/olap/indexes_ut.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -302,9 +302,8 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) {
}

AFL_VERIFY(updatesCount + 6 ==
(ui64)csController->GetActualizationRefreshSchemeCount().Val())(
"updates", updatesCount)("count",
csController->GetActualizationRefreshSchemeCount().Val());
(ui64)csController->GetActualizationRefreshSchemeCount().Val())("updates", updatesCount)(
"count", csController->GetActualizationRefreshSchemeCount().Val());
}

class TTestIndexesScenario {
Expand Down Expand Up @@ -340,6 +339,17 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) {
auto alterResult = session.ExecuteSchemeQuery(alterQuery).GetValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), NYdb::EStatus::SUCCESS, alterResult.GetIssues().ToString());
}
{
auto alterQuery =
TStringBuilder() << Sprintf(
R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=index_ngramm_uid, TYPE=BLOOM_NGRAMM_FILTER,
FEATURES=`{"column_name" : "resource_id", "ngramm_size" : 3, "hashes_count" : 2, "filter_size_bytes" : 64024}`);
)",
StorageId.data());
auto session = tableClient.CreateSession().GetValueSync().GetSession();
auto alterResult = session.ExecuteSchemeQuery(alterQuery).GetValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), NYdb::EStatus::SUCCESS, alterResult.GetIssues().ToString());
}
{
auto alterQuery =
TStringBuilder() << Sprintf(
Expand Down Expand Up @@ -416,6 +426,27 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) {
// important checker for control compactions (<=21) and control indexes constructed (>=21)
AFL_VERIFY(csController->GetCompactionStartedCounter().Val() == 21)("count", csController->GetCompactionStartedCounter().Val());

{
auto it = tableClient
.StreamExecuteScanQuery(R"(
--!syntax_v1
SELECT
COUNT(*)
FROM `/Root/olapStore/olapTable`
WHERE resource_id LIKE '%110a151%'
)")
.GetValueSync();

UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
TString result = StreamResultToYson(it);
AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("result", result);
AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("skip", csController->GetIndexesSkippingOnSelect().Val())(
"check", csController->GetIndexesApprovedOnSelect().Val());
CompareYson(result, R"([[0u;]])");
AFL_VERIFY(!csController->GetIndexesApprovedOnSelect().Val());
AFL_VERIFY(csController->GetIndexesSkippingOnSelect().Val());
}
{
auto it = tableClient
.StreamExecuteScanQuery(R"(
Expand All @@ -441,30 +472,56 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) {
}
AFL_VERIFY(csController->GetIndexesApprovedOnSelect().Val() < csController->GetIndexesSkippingOnSelect().Val());
}
ui32 requestsCount = 100;
for (ui32 i = 0; i < requestsCount; ++i) {
const ui32 idx = RandomNumber<ui32>(uids.size());
const auto query = [](const TString& res, const TString& uid, const ui32 level) {
TStringBuilder sb;
sb << "SELECT COUNT(*) FROM `/Root/olapStore/olapTable`" << Endl;
sb << "WHERE(" << Endl;
sb << "resource_id = '" << res << "' AND" << Endl;
sb << "uid= '" << uid << "' AND" << Endl;
sb << "level= " << level << Endl;
sb << ")";
return sb;
};
auto it = tableClient.StreamExecuteScanQuery(query(resourceIds[idx], uids[idx], levels[idx])).GetValueSync();

UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
TString result = StreamResultToYson(it);
Cout << csController->GetIndexesSkippingOnSelect().Val() << " / " << csController->GetIndexesApprovedOnSelect().Val() << " / "
<< csController->GetIndexesSkippedNoData().Val() << Endl;
CompareYson(result, R"([[1u;]])");
{
ui32 requestsCount = 100;
for (ui32 i = 0; i < requestsCount; ++i) {
const ui32 idx = RandomNumber<ui32>(uids.size());
const auto query = [](const TString& res, const TString& uid, const ui32 level) {
TStringBuilder sb;
sb << "SELECT COUNT(*) FROM `/Root/olapStore/olapTable`" << Endl;
sb << "WHERE(" << Endl;
sb << "resource_id = '" << res << "' AND" << Endl;
sb << "uid= '" << uid << "' AND" << Endl;
sb << "level= " << level << Endl;
sb << ")";
return sb;
};
auto it = tableClient.StreamExecuteScanQuery(query(resourceIds[idx], uids[idx], levels[idx])).GetValueSync();

UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
TString result = StreamResultToYson(it);
Cout << csController->GetIndexesSkippingOnSelect().Val() << " / " << csController->GetIndexesApprovedOnSelect().Val()
<< " / " << csController->GetIndexesSkippedNoData().Val() << Endl;
CompareYson(result, R"([[1u;]])");
}
AFL_VERIFY(csController->GetIndexesApprovedOnSelect().Val() * 5 < csController->GetIndexesSkippingOnSelect().Val())
("approved", csController->GetIndexesApprovedOnSelect().Val())("skipped", csController->GetIndexesSkippingOnSelect().Val());
}
{
const ui64 skipStart = csController->GetIndexesSkippingOnSelect().Val();
const ui64 approveStart = csController->GetIndexesApprovedOnSelect().Val();
ui32 requestsCount = 100;
for (ui32 i = 0; i < requestsCount; ++i) {
const ui32 idx = RandomNumber<ui32>(uids.size());
const auto query = [](const TString& res, const TString& uid, const ui32 level) {
TStringBuilder sb;
sb << "SELECT COUNT(*) FROM `/Root/olapStore/olapTable`" << Endl;
sb << "WHERE" << Endl;
sb << "resource_id LIKE '%" << res << "%'" << Endl;
return sb;
};
auto it = tableClient.StreamExecuteScanQuery(query(resourceIds[idx], uids[idx], levels[idx])).GetValueSync();

UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
TString result = StreamResultToYson(it);
Cout << resourceIds[idx] << "/" << csController->GetIndexesSkippingOnSelect().Val() - skipStart << " / "
<< csController->GetIndexesApprovedOnSelect().Val() - approveStart << " / "
<< csController->GetIndexesSkippedNoData().Val() << Endl;
CompareYson(result, R"([[1u;]])");
}
AFL_VERIFY(csController->GetIndexesSkippingOnSelect().Val() - skipStart > 1)("approved", csController->GetIndexesApprovedOnSelect().Val() - approveStart)(
"skipped", csController->GetIndexesSkippingOnSelect().Val() - skipStart);
}

AFL_VERIFY(csController->GetIndexesApprovedOnSelect().Val() * 5 < csController->GetIndexesSkippingOnSelect().Val())
("approved", csController->GetIndexesApprovedOnSelect().Val())("skipped", csController->GetIndexesSkippingOnSelect().Val());
}
};

Expand Down
16 changes: 16 additions & 0 deletions ydb/core/protos/flat_scheme_op.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -391,6 +391,13 @@ message TRequestedBloomFilter {
repeated string ColumnNames = 3;
}

message TRequestedBloomNGrammFilter {
optional uint32 NGrammSize = 1;
optional uint32 FilterSizeBytes = 2;
optional uint32 HashesCount = 3;
optional string ColumnName = 4;
}

message TRequestedMaxIndex {
optional string ColumnName = 1;
}
Expand All @@ -410,6 +417,7 @@ message TOlapIndexRequested {
TRequestedBloomFilter BloomFilter = 40;
TRequestedMaxIndex MaxIndex = 41;
TRequestedCountMinSketch CountMinSketch = 42;
TRequestedBloomNGrammFilter BloomNGrammFilter = 43;
}
}

Expand All @@ -419,6 +427,13 @@ message TBloomFilter {
repeated uint32 ColumnIds = 3;
}

message TBloomNGrammFilter {
optional uint32 NGrammSize = 1;
optional uint32 FilterSizeBytes = 2;
optional uint32 HashesCount = 3;
optional uint32 ColumnId = 4;
}

message TMaxIndex {
optional uint32 ColumnId = 1;
}
Expand All @@ -441,6 +456,7 @@ message TOlapIndexDescription {
TBloomFilter BloomFilter = 41;
TMaxIndex MaxIndex = 42;
TCountMinSketch CountMinSketch = 43;
TBloomNGrammFilter BloomNGrammFilter = 44;
}
}

Expand Down
209 changes: 0 additions & 209 deletions ydb/core/protos/ssa.proto
View file

This file was deleted.

Loading

0 comments on commit 4623dc8

Please sign in to comment.