Skip to content

Commit

Permalink
Add some comments about decoding
Browse files Browse the repository at this point in the history 
Signed-off-by: JaySon-Huang <jayson.hjs@gmail.com>
  • Loading branch information
@JaySon-Huang
JaySon-Huang committed Jun 17, 2022
1 parent 617fe54 commit 70df8f2
Show file tree
Hide file tree
Showing 10 changed files with 160 additions and 91 deletions.
46 changes: 23 additions & 23 deletions dbms/src/Debug/dbgFuncMockRaftSnapshot.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ RegionPtr GenDbgRegionSnapshotWithData(Context & context, const ASTs & args)
{
const String & database_name = typeid_cast<const ASTIdentifier &>(*args[0]).name;
const String & table_name = typeid_cast<const ASTIdentifier &>(*args[1]).name;
RegionID region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[2]).value));
auto region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[2]).value));
TableID table_id = RegionBench::getTableID(context, database_name, table_name, "");
MockTiDB::TablePtr table = MockTiDB::instance().getTableByName(database_name, table_name);
auto & table_info = table->table_info;
Expand All @@ -70,8 +70,8 @@ RegionPtr GenDbgRegionSnapshotWithData(Context & context, const ASTs & args)

if (!is_common_handle)
{
HandleID start = static_cast<HandleID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[3]).value));
HandleID end = static_cast<HandleID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[4]).value));
auto start = static_cast<HandleID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[3]).value));
auto end = static_cast<HandleID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[4]).value));
region = RegionBench::createRegion(table_id, region_id, start, end);
}
else
Expand Down Expand Up @@ -105,8 +105,8 @@ RegionPtr GenDbgRegionSnapshotWithData(Context & context, const ASTs & args)
for (auto it = args_begin; it != args_end; it += len)
{
HandleID handle_id = is_common_handle ? 0 : static_cast<HandleID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*it[0]).value));
Timestamp tso = static_cast<Timestamp>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*it[1]).value));
UInt8 del = static_cast<UInt8>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*it[2]).value));
auto tso = static_cast<Timestamp>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*it[1]).value));
auto del = static_cast<UInt8>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*it[2]).value));
{
std::vector<Field> fields;

Expand Down Expand Up @@ -168,7 +168,7 @@ void MockRaftCommand::dbgFuncRegionSnapshotWithData(Context & context, const AST
// DBGInvoke region_snapshot(region-id, start-key, end-key, database-name, table-name[, partition-id])
void MockRaftCommand::dbgFuncRegionSnapshot(Context & context, const ASTs & args, DBGInvoker::Printer output)
{
RegionID region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[0]).value));
auto region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[0]).value));
bool has_partition_id = false;
size_t args_size = args.size();
if (dynamic_cast<ASTLiteral *>(args[args_size - 1].get()) != nullptr)
Expand Down Expand Up @@ -214,15 +214,15 @@ void MockRaftCommand::dbgFuncRegionSnapshot(Context & context, const ASTs & args
}
else
{
HandleID start = static_cast<HandleID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[1]).value));
HandleID end = static_cast<HandleID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[2]).value));
auto start = static_cast<HandleID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[1]).value));
auto end = static_cast<HandleID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[2]).value));
start_key = RecordKVFormat::genKey(table_id, start);
end_key = RecordKVFormat::genKey(table_id, end);
}
region_info.set_start_key(start_key.toString());
region_info.set_end_key(end_key.toString());
*region_info.add_peers() = createPeer(1, true);
*region_info.add_peers() = createPeer(2, true);
*region_info.add_peers() = tests::createPeer(1, true);
*region_info.add_peers() = tests::createPeer(2, true);
auto peer_id = 1;
auto start_decoded_key = RecordKVFormat::decodeTiKVKey(start_key);
auto end_decoded_key = RecordKVFormat::decodeTiKVKey(end_key);
Expand Down Expand Up @@ -432,9 +432,9 @@ void MockRaftCommand::dbgFuncIngestSST(Context & context, const ASTs & args, DBG
{
const String & database_name = typeid_cast<const ASTIdentifier &>(*args[0]).name;
const String & table_name = typeid_cast<const ASTIdentifier &>(*args[1]).name;
RegionID region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[2]).value));
RegionID start_handle = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[3]).value));
RegionID end_handle = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[4]).value));
auto region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[2]).value));
auto start_handle = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[3]).value));
auto end_handle = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[4]).value));
MockTiDB::TablePtr table = MockTiDB::instance().getTableByName(database_name, table_name);

const auto & table_info = RegionBench::getTableInfo(context, database_name, table_name);
Expand Down Expand Up @@ -555,7 +555,7 @@ void MockRaftCommand::dbgFuncRegionSnapshotApplyBlock(Context & context, const A
throw Exception("Args not matched, should be: region-id", ErrorCodes::BAD_ARGUMENTS);
}

RegionID region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args.front()).value));
auto region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args.front()).value));
auto [region, block_cache] = GLOBAL_REGION_MAP.popRegionCache("__snap_" + std::to_string(region_id));
auto & tmt = context.getTMTContext();
context.getTMTContext().getKVStore()->checkAndApplySnapshot<RegionPtrWithBlock>({region, std::move(block_cache)}, tmt);
Expand All @@ -577,12 +577,12 @@ void MockRaftCommand::dbgFuncRegionSnapshotPreHandleDTFiles(Context & context, c

const String & database_name = typeid_cast<const ASTIdentifier &>(*args[0]).name;
const String & table_name = typeid_cast<const ASTIdentifier &>(*args[1]).name;
RegionID region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[2]).value));
RegionID start_handle = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[3]).value));
RegionID end_handle = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[4]).value));
auto region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[2]).value));
auto start_handle = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[3]).value));
auto end_handle = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[4]).value));

const String schema_str = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[5]).value);
String handle_pk_name = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[6]).value);
const auto schema_str = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[5]).value);
auto handle_pk_name = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[6]).value);

UInt64 test_fields = 1;
if (args.size() > 7)
Expand Down Expand Up @@ -677,10 +677,10 @@ void MockRaftCommand::dbgFuncRegionSnapshotPreHandleDTFilesWithHandles(Context &

const String & database_name = typeid_cast<const ASTIdentifier &>(*args[0]).name;
const String & table_name = typeid_cast<const ASTIdentifier &>(*args[1]).name;
RegionID region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[2]).value));
auto region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[2]).value));

const String schema_str = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[3]).value);
String handle_pk_name = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[4]).value);
const auto schema_str = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[3]).value);
auto handle_pk_name = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[4]).value);

std::vector<UInt64> handles;
for (size_t i = 5; i < args.size(); ++i)
Expand Down Expand Up @@ -770,7 +770,7 @@ void MockRaftCommand::dbgFuncRegionSnapshotApplyDTFiles(Context & context, const
if (args.size() != 1)
throw Exception("Args not matched, should be: region-id", ErrorCodes::BAD_ARGUMENTS);

RegionID region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args.front()).value));
auto region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args.front()).value));
const auto region_name = "__snap_snap_" + std::to_string(region_id);
auto [new_region, ingest_ids] = GLOBAL_REGION_MAP.popRegionSnap(region_name);
auto & tmt = context.getTMTContext();
Expand Down
4 changes: 4 additions & 0 deletions dbms/src/Storages/Page/V3/tests/gtest_blob_store.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -306,20 +306,23 @@ try
blob_store.blob_stats.restoreByEntry(PageEntryV3{
.file_id = file_id1,
.size = 128,
.padded_size = 0,
.tag = 0,
.offset = 1024,
.checksum = 0x4567,
});
blob_store.blob_stats.restoreByEntry(PageEntryV3{
.file_id = file_id1,
.size = 512,
.padded_size = 0,
.tag = 0,
.offset = 2048,
.checksum = 0x4567,
});
blob_store.blob_stats.restoreByEntry(PageEntryV3{
.file_id = file_id2,
.size = 512,
.padded_size = 0,
.tag = 0,
.offset = 2048,
.checksum = 0x4567,
Expand Down Expand Up @@ -402,6 +405,7 @@ try
blob_store.blob_stats.restoreByEntry(PageEntryV3{
.file_id = id,
.size = 1024,
.padded_size = 0,
.tag = 0,
.offset = 0,
.checksum = 0x4567,
Expand Down
4 changes: 2 additions & 2 deletions dbms/src/Storages/Transaction/DecodingStorageSchemaSnapshot.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,11 +125,11 @@ struct DecodingStorageSchemaSnapshot
{
auto pk_pos_iter = pk_pos_map.begin();
size_t column_pos_in_block = 0;
for (auto iter = sorted_column_id_with_pos.begin(); iter != sorted_column_id_with_pos.end(); iter++)
for (auto & column_id_with_pos : sorted_column_id_with_pos)
{
if (pk_pos_iter == pk_pos_map.end())
break;
if (pk_pos_iter->first == iter->first)
if (pk_pos_iter->first == column_id_with_pos.first)
{
pk_pos_iter->second = column_pos_in_block;
pk_pos_iter++;
Expand Down
1 change: 1 addition & 0 deletions dbms/src/Storages/Transaction/PartitionStreams.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
// limitations under the License.

#include <Common/Allocator.h>
#include <Common/Exception.h>
#include <Common/FailPoint.h>
#include <Common/TiFlashMetrics.h>
#include <Common/setThreadName.h>
Expand Down
1 change: 1 addition & 0 deletions dbms/src/Storages/Transaction/RegionBlockReader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,7 @@ bool RegionBlockReader::readImpl(Block & block, const RegionDataReadInfoList & d
}
else
{
// For common handle, sometimes we need to decode the value from encoded key instead of encoded value
auto * raw_extra_column = const_cast<IColumn *>((block.getByPosition(extra_handle_column_pos)).column.get());
raw_extra_column->insertData(pk->data(), pk->size());
/// decode key and insert pk columns if needed
Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Storages/Transaction/RegionBlockReader.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ class Block;
class RegionBlockReader : private boost::noncopyable
{
public:
RegionBlockReader(DecodingStorageSchemaSnapshotConstPtr schema_snapshot_);
explicit RegionBlockReader(DecodingStorageSchemaSnapshotConstPtr schema_snapshot_);

/// Read `data_list` as a block.
///
Expand Down
43 changes: 26 additions & 17 deletions dbms/src/Storages/Transaction/RowCodec.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -314,7 +314,7 @@ bool appendRowV2ToBlock(
ColumnID pk_handle_id,
bool force_decode)
{
UInt8 row_flag = readLittleEndian<UInt8>(&raw_value[1]);
auto row_flag = readLittleEndian<UInt8>(&raw_value[1]);
bool is_big = row_flag & RowV2::BigRowMask;
return is_big ? appendRowV2ToBlockImpl<true>(raw_value, column_ids_iter, column_ids_iter_end, block, block_column_pos, column_infos, pk_handle_id, force_decode)
: appendRowV2ToBlockImpl<false>(raw_value, column_ids_iter, column_ids_iter_end, block, block_column_pos, column_infos, pk_handle_id, force_decode);
Expand Down Expand Up @@ -360,9 +360,10 @@ bool appendRowV2ToBlockImpl(
decodeUInts<ColumnID, typename RowV2::Types<is_big>::ColumnIDType>(cursor, raw_value, num_null_columns, null_column_ids);
decodeUInts<size_t, typename RowV2::Types<is_big>::ValueOffsetType>(cursor, raw_value, num_not_null_columns, value_offsets);
size_t values_start_pos = cursor;
size_t id_not_null = 0, id_null = 0;
size_t idx_not_null = 0;
size_t idx_null = 0;
// Merge ordered not null/null columns to keep order.
while (id_not_null < not_null_column_ids.size() || id_null < null_column_ids.size())
while (idx_not_null < not_null_column_ids.size() || idx_null < null_column_ids.size())
{
if (column_ids_iter == column_ids_iter_end)
{
Expand All @@ -371,24 +372,31 @@ bool appendRowV2ToBlockImpl(
}

bool is_null;
if (id_not_null < not_null_column_ids.size() && id_null < null_column_ids.size())
is_null = not_null_column_ids[id_not_null] > null_column_ids[id_null];
if (idx_not_null < not_null_column_ids.size() && idx_null < null_column_ids.size())
is_null = not_null_column_ids[idx_not_null] > null_column_ids[idx_null];
else
is_null = id_null < null_column_ids.size();
is_null = idx_null < null_column_ids.size();

auto next_datum_column_id = is_null ? null_column_ids[id_null] : not_null_column_ids[id_not_null];
auto next_datum_column_id = is_null ? null_column_ids[idx_null] : not_null_column_ids[idx_not_null];
if (column_ids_iter->first > next_datum_column_id)
{
// extra column
// The next column id to read is bigger than the column id of next datum in encoded row.
// It means this is the datum of extra column. May happen when reading after dropping
// a column.
if (!force_decode)
return false;
// Ignore the extra column and continue to parse other datum
if (is_null)
id_null++;
idx_null++;
else
id_not_null++;
idx_not_null++;
}
else if (column_ids_iter->first < next_datum_column_id)
{
// The next column id to read is less than the column id of next datum in encoded row.
// It means this is the datum of missing column. May happen when reading after adding
// a column.
// Fill with default value and continue to read data for next column id.
const auto & column_info = column_infos[column_ids_iter->second];
if (!addDefaultValueToColumnIfPossible(column_info, block, block_column_pos, force_decode))
return false;
Expand All @@ -397,23 +405,24 @@ bool appendRowV2ToBlockImpl(
}
else
{
// if pk_handle_id is a valid column id, then it means the table's pk_is_handle is true
// If pk_handle_id is a valid column id, then it means the table's pk_is_handle is true
// we can just ignore the pk value encoded in value part
if (unlikely(column_ids_iter->first == pk_handle_id))
{
column_ids_iter++;
block_column_pos++;
if (is_null)
{
id_null++;
idx_null++;
}
else
{
id_not_null++;
idx_not_null++;
}
continue;
}

// Parse the datum.
auto * raw_column = const_cast<IColumn *>((block.getByPosition(block_column_pos)).column.get());
const auto & column_info = column_infos[column_ids_iter->second];
if (is_null)
Expand All @@ -432,15 +441,15 @@ bool appendRowV2ToBlockImpl(
}
// ColumnNullable::insertDefault just insert a null value
raw_column->insertDefault();
id_null++;
idx_null++;
}
else
{
size_t start = id_not_null ? value_offsets[id_not_null - 1] : 0;
size_t length = value_offsets[id_not_null] - start;
size_t start = idx_not_null ? value_offsets[idx_not_null - 1] : 0;
size_t length = value_offsets[idx_not_null] - start;
if (!raw_column->decodeTiDBRowV2Datum(values_start_pos + start, raw_value, length, force_decode))
return false;
id_not_null++;
idx_not_null++;
}
column_ids_iter++;
block_column_pos++;
Expand Down
Loading

0 comments on commit 70df8f2

Please sign in to comment.