Skip to content

Commit

Permalink
KIKIMR-19521 BTreeIndex Charge Interface
Browse files Browse the repository at this point in the history
  • Loading branch information
kunga committed Dec 14, 2023
1 parent 897ada7 commit e7f64cc
Show file tree
Hide file tree
Showing 17 changed files with 325 additions and 167 deletions.
2 changes: 2 additions & 0 deletions ydb/core/tablet_flat/CMakeLists.darwin-arm64.txt
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,8 @@ target_sources(ydb-core-tablet_flat PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_mem_warm.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_sausagecache.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_sausage_meta.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_part_charge_create.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_part_charge_range.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_page_label.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_part_dump.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_part_iter_multi.cpp
Expand Down
2 changes: 2 additions & 0 deletions ydb/core/tablet_flat/CMakeLists.darwin-x86_64.txt
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,8 @@ target_sources(ydb-core-tablet_flat PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_mem_warm.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_sausagecache.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_sausage_meta.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_part_charge_create.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_part_charge_range.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_page_label.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_part_dump.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_part_iter_multi.cpp
Expand Down
2 changes: 2 additions & 0 deletions ydb/core/tablet_flat/CMakeLists.linux-aarch64.txt
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,8 @@ target_sources(ydb-core-tablet_flat PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_mem_warm.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_sausagecache.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_sausage_meta.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_part_charge_create.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_part_charge_range.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_page_label.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_part_dump.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_part_iter_multi.cpp
Expand Down
2 changes: 2 additions & 0 deletions ydb/core/tablet_flat/CMakeLists.linux-x86_64.txt
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,8 @@ target_sources(ydb-core-tablet_flat PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_mem_warm.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_sausagecache.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_sausage_meta.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_part_charge_create.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_part_charge_range.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_page_label.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_part_dump.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_part_iter_multi.cpp
Expand Down
2 changes: 2 additions & 0 deletions ydb/core/tablet_flat/CMakeLists.windows-x86_64.txt
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,8 @@ target_sources(ydb-core-tablet_flat PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_mem_warm.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_sausagecache.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_sausage_meta.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_part_charge_create.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_part_charge_range.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_page_label.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_part_dump.cpp
${CMAKE_SOURCE_DIR}/ydb/core/tablet_flat/flat_part_iter_multi.cpp
Expand Down
8 changes: 4 additions & 4 deletions ydb/core/tablet_flat/benchmark/b_charge.cpp
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
#include <benchmark/benchmark.h>
#include <random>

#include <ydb/core/tablet_flat/flat_row_celled.h>
#include <ydb/core/tablet_flat/flat_part_charge.h>
#include <ydb/core/tablet_flat/flat_part_charge_range.h>
#include <ydb/core/tablet_flat/flat_part_charge_create.h>
#include <ydb/core/tablet_flat/test/libs/rows/cook.h>
#include <ydb/core/tablet_flat/test/libs/rows/tool.h>
#include <ydb/core/tablet_flat/test/libs/table/model/large.h>
Expand Down Expand Up @@ -138,7 +138,7 @@ BENCHMARK_DEFINE_F(TModel, PrechargeByKeys)(benchmark::State& state) {
const auto from = Tool.KeyCells(Mass.Saved[lower]);
const auto to = Tool.KeyCells(Mass.Saved[upper]);

TCharge::Range(Env.Get(), from, to, *Run.Get(), keyDefaults, Tags, items, Max<ui64>());
ChargeRange(Env.Get(), from, to, *Run.Get(), keyDefaults, Tags, items, Max<ui64>());
}

state.counters["Touched"] = benchmark::Counter(Env->TouchedCount, benchmark::Counter::kAvgIterations);
Expand All @@ -154,7 +154,7 @@ BENCHMARK_DEFINE_F(TModel, PrechargeByRows)(benchmark::State& state) {
ui32 lower = ++it % 50;
ui32 upper = lower + items;

TCharge(Env.Get(), *(Run.Get())->begin()->Part, Tags, false).Do(lower, upper, keyDefaults, items, Max<ui64>());
CreateCharge(Env.Get(), *(Run.Get())->begin()->Part, Tags, false)->Do(lower, upper, keyDefaults, items, Max<ui64>());
}

state.counters["Touched"] = Env->TouchedCount / it;
Expand Down
153 changes: 6 additions & 147 deletions ydb/core/tablet_flat/flat_part_charge.h
Original file line number Diff line number Diff line change
@@ -1,27 +1,20 @@
#pragma once

#include "defs.h"
#include "flat_table_part.h"
#include "flat_part_iface.h"
#include "flat_part_slice.h"
#include "flat_part_index_iter.h"
#include "flat_part_charge_iface.h"

#include <util/generic/bitmap.h>

namespace NKikimr {
namespace NTable {

class TCharge {
class TCharge : public ICharge {
public:
using TCells = NPage::TCells;
using TIter = NPage::TIndex::TIter;
using TDataPage = NPage::TDataPage;
using TGroupId = NPage::TGroupId;

struct TResult {
bool Ready; /* All required pages are already in memory */
bool Overshot; /* Search may start outside of bounds */
};

TCharge(IPages *env, const TPart &part, TTagsRef tags, bool includeHistory = false)
: Env(env)
Expand Down Expand Up @@ -49,131 +42,8 @@ namespace NTable {
}
}

static bool Range(IPages *env, const TCells key1, const TCells key2,
const TRun &run, const TKeyCellDefaults &keyDefaults, TTagsRef tags,
ui64 items, ui64 bytes, bool includeHistory = false) noexcept
{
if (run.size() == 1) {
auto pos = run.begin();
TRowId row1 = pos->Slice.BeginRowId();
TRowId row2 = pos->Slice.EndRowId() - 1;
return TCharge(env, *pos->Part, tags, includeHistory).Do(key1, key2, row1, row2, keyDefaults, items, bytes).Ready;
}

bool ready = true;
auto pos = run.LowerBound(key1);

if (pos == run.end())
return true;

bool fromStart = TSlice::CompareSearchKeyFirstKey(key1, pos->Slice, keyDefaults) <= 0;

while (pos != run.end()) {
TRowId row1 = pos->Slice.BeginRowId();
TRowId row2 = pos->Slice.EndRowId() - 1;

const int cmp = TSlice::CompareLastKeySearchKey(pos->Slice, key2, keyDefaults);

TArrayRef<const TCell> key1r;
if (!fromStart) {
key1r = key1;
}
TArrayRef<const TCell> key2r;
if (cmp > 0 /* slice->LastKey > key2 */) {
key2r = key2;
}

auto r = TCharge(env, *pos->Part, tags, includeHistory).Do(key1r, key2r, row1, row2, keyDefaults, items, bytes);
ready &= r.Ready;

if (cmp >= 0 /* slice->LastKey >= key2 */) {
if (r.Overshot && ++pos != run.end()) {
// Unfortunately key > key2 might be at the start of the next slice
TRowId firstRow = pos->Slice.BeginRowId();
// Precharge the first row on the next slice
TCharge(env, *pos->Part, tags, includeHistory).Do(firstRow, firstRow, keyDefaults, items, bytes);
}

break;
}

// Will consume this slice before encountering key2
fromStart = true;
++pos;
}

return ready;
}

static bool RangeReverse(IPages *env, const TCells key1, const TCells key2,
const TRun &run, const TKeyCellDefaults &keyDefaults, TTagsRef tags,
ui64 items, ui64 bytes, bool includeHistory = false) noexcept
{
if (run.size() == 1) {
auto pos = run.begin();
TRowId row1 = pos->Slice.EndRowId() - 1;
TRowId row2 = pos->Slice.BeginRowId();
return TCharge(env, *pos->Part, tags, includeHistory).DoReverse(key1, key2, row1, row2, keyDefaults, items, bytes).Ready;
}

bool ready = true;
auto pos = run.LowerBoundReverse(key1);

if (pos == run.end())
return true;

bool fromEnd = TSlice::CompareLastKeySearchKey(pos->Slice, key1, keyDefaults) <= 0;

for (;;) {
TRowId row1 = pos->Slice.EndRowId() - 1;
TRowId row2 = pos->Slice.BeginRowId();

// N.B. empty key2 is like -inf during reverse iteration
const int cmp = key2 ? TSlice::CompareSearchKeyFirstKey(key2, pos->Slice, keyDefaults) : -1;

TArrayRef<const TCell> key1r;
if (!fromEnd) {
key1r = key1;
}
TArrayRef<const TCell> key2r;
if (cmp > 0 /* key2 > slice->FirstKey */) {
key2r = key2;
}

auto r = TCharge(env, *pos->Part, tags, includeHistory).DoReverse(key1r, key2r, row1, row2, keyDefaults, items, bytes);
ready &= r.Ready;

if (pos == run.begin()) {
break;
}

if (cmp >= 0 /* key2 >= slice->FirstKey */) {
if (r.Overshot) {
--pos;
// Unfortunately key <= key2 might be at the end of the previous slice
TRowId lastRow = pos->Slice.EndRowId() - 1;
// Precharge the last row on the previous slice
TCharge(env, *pos->Part, tags, includeHistory).DoReverse(lastRow, lastRow, keyDefaults, items, bytes);
}

break;
}

// Will consume this slice before encountering key2
fromEnd = true;
--pos;
}

return ready;
}

/**
* Precharges data for rows between row1 and row2 inclusive
*
* Important caveat: assumes iteration won't touch any row > row2
*/
bool Do(const TRowId row1, const TRowId row2,
const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, ui64 bytesLimit) const noexcept
const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, ui64 bytesLimit) const noexcept override
{
auto index = Index.TryLoadRaw();
if (!index) {
Expand All @@ -199,13 +69,8 @@ namespace NTable {
return DoPrecharge(TCells{}, TCells{}, TIter{}, TIter{}, first, last, startRow, endRow, keyDefaults, itemsLimit, bytesLimit);
}

/**
* Precharges data for rows between row1 and row2 inclusive in reverse
*
* Important caveat: assumes iteration won't touch any row > row2
*/
bool DoReverse(const TRowId row1, const TRowId row2,
const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, ui64 bytesLimit) const noexcept
const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, ui64 bytesLimit) const noexcept override
{
auto index = Index.TryLoadRaw();
if (!index) {
Expand Down Expand Up @@ -236,12 +101,9 @@ namespace NTable {
return DoPrechargeReverse(TCells{}, TCells{}, TIter{}, TIter{}, first, last, startRow, endRow, keyDefaults, itemsLimit, bytesLimit);
}

/**
* Precharges data for rows between max(key1, row1) and min(key2, row2) inclusive
*/
TResult Do(const TCells key1, const TCells key2, const TRowId row1,
const TRowId row2, const TKeyCellDefaults &keyDefaults, ui64 itemsLimit,
ui64 bytesLimit) const noexcept
ui64 bytesLimit) const noexcept override
{
auto index = Index.TryLoadRaw();
if (!index) {
Expand Down Expand Up @@ -307,12 +169,9 @@ namespace NTable {
return { ready, overshot };
}

/**
* Precharges data for rows between min(key1, row1) and max(key2, row2) inclusive in reverse
*/
TResult DoReverse(const TCells key1, const TCells key2, const TRowId row1,
const TRowId row2, const TKeyCellDefaults &keyDefaults, ui64 itemsLimit,
ui64 bytesLimit) const noexcept
ui64 bytesLimit) const noexcept override
{
auto index = Index.TryLoadRaw();
if (!index) {
Expand Down
69 changes: 69 additions & 0 deletions ydb/core/tablet_flat/flat_part_charge_btree_index.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
#pragma once

#include "flat_table_part.h"
#include "flat_part_iface.h"
#include "flat_part_charge_iface.h"

namespace NKikimr::NTable {

class TChargeBTreeIndex : public ICharge {
public:
TChargeBTreeIndex(IPages *env, const TPart &part, TTagsRef tags, bool includeHistory = false) {
Y_UNUSED(env);
Y_UNUSED(part);
Y_UNUSED(tags);
Y_UNUSED(includeHistory);
}

virtual bool Do(const TRowId row1, const TRowId row2,
const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, ui64 bytesLimit) const noexcept override {
// TODO: implement
Y_UNUSED(row1);
Y_UNUSED(row2);
Y_UNUSED(keyDefaults);
Y_UNUSED(itemsLimit);
Y_UNUSED(bytesLimit);
return true;
}

virtual bool DoReverse(const TRowId row1, const TRowId row2,
const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, ui64 bytesLimit) const noexcept override {
// TODO: implement
Y_UNUSED(row1);
Y_UNUSED(row2);
Y_UNUSED(keyDefaults);
Y_UNUSED(itemsLimit);
Y_UNUSED(bytesLimit);
return true;
}

virtual TResult Do(const TCells key1, const TCells key2, const TRowId row1,
const TRowId row2, const TKeyCellDefaults &keyDefaults, ui64 itemsLimit,
ui64 bytesLimit) const noexcept override {
// TODO: implement
Y_UNUSED(key1);
Y_UNUSED(key2);
Y_UNUSED(row1);
Y_UNUSED(row2);
Y_UNUSED(keyDefaults);
Y_UNUSED(itemsLimit);
Y_UNUSED(bytesLimit);
return {true, false};
}

virtual TResult DoReverse(const TCells key1, const TCells key2, const TRowId row1,
const TRowId row2, const TKeyCellDefaults &keyDefaults, ui64 itemsLimit,
ui64 bytesLimit) const noexcept override {
// TODO: implement
Y_UNUSED(key1);
Y_UNUSED(key2);
Y_UNUSED(row1);
Y_UNUSED(row2);
Y_UNUSED(keyDefaults);
Y_UNUSED(itemsLimit);
Y_UNUSED(bytesLimit);
return {true, false};
}
};

}
15 changes: 15 additions & 0 deletions ydb/core/tablet_flat/flat_part_charge_create.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#include "flat_part_charge_create.h"
#include "flat_part_charge.h"
#include "flat_part_charge_btree_index.h"

namespace NKikimr::NTable {

THolder<ICharge> CreateCharge(IPages *env, const TPart &part, TTagsRef tags, bool includeHistory) {
if (part.IndexPages.BTreeGroups) {
return MakeHolder<TChargeBTreeIndex>(env, part, tags, includeHistory);
} else {
return MakeHolder<TCharge>(env, part, tags, includeHistory);
}
}

}
10 changes: 10 additions & 0 deletions ydb/core/tablet_flat/flat_part_charge_create.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#pragma once

#include "flat_part_charge_iface.h"
#include "flat_part_iface.h"

namespace NKikimr::NTable {

THolder<ICharge> CreateCharge(IPages *env, const TPart &part, TTagsRef tags, bool includeHistory = false);

}
Loading

0 comments on commit e7f64cc

Please sign in to comment.