Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[cherry-pick](branch-2.1) support decimal256 for parquet reader #42241

Merged
merged 2 commits into from
Oct 22, 2024
Merged

[cherry-pick](branch-2.1) support decimal256 for parquet reader #42241

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4d38546
[enhance](parquet) support decimal256 for parquet reader (#41526)
suxiaogang223 Oct 9, 2024
34fa6ee
fix build
suxiaogang223 Oct 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions be/src/gutil/endian.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,8 @@ inline unsigned __int128 gbswap_128(unsigned __int128 host_int) {
}

inline wide::UInt256 gbswap_256(wide::UInt256 host_int) {
wide::UInt256 result{gbswap_64(host_int.items[3]), gbswap_64(host_int.items[2]),
gbswap_64(host_int.items[1]), gbswap_64(host_int.items[0])};
wide::UInt256 result {gbswap_64(host_int.items[3]), gbswap_64(host_int.items[2]),
gbswap_64(host_int.items[1]), gbswap_64(host_int.items[0])};
return result;
}

Expand Down Expand Up @@ -137,6 +137,9 @@ class LittleEndian {
static unsigned __int128 FromHost128(unsigned __int128 x) { return x; }
static unsigned __int128 ToHost128(unsigned __int128 x) { return x; }

static wide::UInt256 FromHost256(wide::UInt256 x) { return x; }
static wide::UInt256 ToHost256(wide::UInt256 x) { return x; }

static bool IsLittleEndian() { return true; }

#elif defined IS_BIG_ENDIAN
Expand All @@ -150,6 +153,12 @@ class LittleEndian {
static uint64 FromHost64(uint64 x) { return gbswap_64(x); }
static uint64 ToHost64(uint64 x) { return gbswap_64(x); }

static unsigned __int128 FromHost128(unsigned __int128 x) { return gbswap_128(x); }
static unsigned __int128 ToHost128(unsigned __int128 x) { return gbswap_128(x); }

static wide::UInt256 FromHost256(wide::UInt256 x) { return gbswap_256(x); }
static wide::UInt256 ToHost256(wide::UInt256 x) { return gbswap_256(x); }

static bool IsLittleEndian() { return false; }

#endif /* ENDIAN */
Expand Down
9 changes: 8 additions & 1 deletion be/src/util/bit_util.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@

#pragma once

#include <type_traits>

#include "vec/core/wide_integer.h"
#ifndef __APPLE__
#include <endian.h>
#endif
Expand Down Expand Up @@ -209,7 +212,11 @@ class BitUtil {

template <typename T>
static T big_endian_to_host(T value) {
if constexpr (std::is_same_v<T, __int128>) {
if constexpr (std::is_same_v<T, wide::Int256>) {
return BigEndian::ToHost256(value);
} else if constexpr (std::is_same_v<T, wide::UInt256>) {
return BigEndian::ToHost256(value);
} else if constexpr (std::is_same_v<T, __int128>) {
return BigEndian::ToHost128(value);
} else if constexpr (std::is_same_v<T, unsigned __int128>) {
return BigEndian::ToHost128(value);
Expand Down
1 change: 1 addition & 0 deletions be/src/vec/core/wide_integer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
// and modified by Doris
#pragma once

#include <cstddef>
#include <cstdint>
#include <initializer_list>
#include <limits>
Expand Down
5 changes: 4 additions & 1 deletion be/src/vec/exec/format/parquet/parquet_column_convert.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

#include <cctz/time_zone.h>

#include "runtime/define_primitive_type.h"
#include "vec/columns/column_nullable.h"
namespace doris::vectorized::parquet {
const cctz::time_zone ConvertParams::utc0 = cctz::utc_time_zone();
Expand All @@ -27,7 +28,8 @@ const cctz::time_zone ConvertParams::utc0 = cctz::utc_time_zone();
M(TYPE_DECIMALV2) \
M(TYPE_DECIMAL32) \
M(TYPE_DECIMAL64) \
M(TYPE_DECIMAL128I)
M(TYPE_DECIMAL128I) \
M(TYPE_DECIMAL256)

bool PhysicalToLogicalConverter::is_parquet_native_type(PrimitiveType type) {
switch (type) {
Expand All @@ -50,6 +52,7 @@ bool PhysicalToLogicalConverter::is_decimal_type(doris::PrimitiveType type) {
case TYPE_DECIMAL32:
case TYPE_DECIMAL64:
case TYPE_DECIMAL128I:
case TYPE_DECIMAL256:
case TYPE_DECIMALV2:
return true;
default:
Expand Down
19 changes: 18 additions & 1 deletion be/src/vec/exec/format/parquet/parquet_column_convert.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include <gen_cpp/parquet_types.h>
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

warning: 'gen_cpp/parquet_types.h' file not found [clang-diagnostic-error]

#include <gen_cpp/parquet_types.h>
         ^


#include "vec/core/types.h"
#include "vec/core/wide_integer.h"
#include "vec/data_types/data_type_factory.hpp"
#include "vec/exec/format/column_type_convert.h"
#include "vec/exec/format/format_common.h"
Expand Down Expand Up @@ -401,7 +402,23 @@ class FixedSizeToDecimal : public PhysicalToLogicalConverter {
M(13, int128_t) \
M(14, int128_t) \
M(15, int128_t) \
M(16, int128_t)
M(16, int128_t) \
M(17, wide::Int256) \
M(18, wide::Int256) \
M(19, wide::Int256) \
M(20, wide::Int256) \
M(21, wide::Int256) \
M(22, wide::Int256) \
M(23, wide::Int256) \
M(24, wide::Int256) \
M(25, wide::Int256) \
M(26, wide::Int256) \
M(27, wide::Int256) \
M(28, wide::Int256) \
M(29, wide::Int256) \
M(30, wide::Int256) \
M(31, wide::Int256) \
M(32, wide::Int256)

switch (_type_length) {
APPLY_FOR_DECIMALS()
Expand Down
19 changes: 18 additions & 1 deletion be/test/util/bit_util_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
#include <gtest/gtest-test-part.h>

#include <boost/utility/binary.hpp>
#include <memory>

#include "gtest/gtest_pred_impl.h"

Expand All @@ -48,4 +47,22 @@ TEST(BitUtil, Popcount) {
EXPECT_EQ(BitUtil::popcount_no_hw(0), 0);
}

TEST(BitUtil, BigEndianToHost) {
uint16_t v16 = 0x1234;
uint32_t v32 = 0x12345678;
uint64_t v64 = 0x123456789abcdef0;
unsigned __int128 v128 = ((__int128)0x123456789abcdef0LL << 64) | 0x123456789abcdef0LL;
wide::UInt256 v256 =
wide::UInt256(0x123456789abcdef0) << 192 | wide::UInt256(0x123456789abcdef0) << 128 |
wide::UInt256(0x123456789abcdef0) << 64 | wide::UInt256(0x123456789abcdef0);
EXPECT_EQ(BitUtil::big_endian_to_host(v16), 0x3412);
EXPECT_EQ(BitUtil::big_endian_to_host(v32), 0x78563412);
EXPECT_EQ(BitUtil::big_endian_to_host(v64), 0xf0debc9a78563412);
EXPECT_EQ(BitUtil::big_endian_to_host(v128),
((__int128)0xf0debc9a78563412LL << 64) | 0xf0debc9a78563412LL);
EXPECT_EQ(BitUtil::big_endian_to_host(v256),
wide::UInt256(0xf0debc9a78563412) << 192 | wide::UInt256(0xf0debc9a78563412) << 128 |
wide::UInt256(0xf0debc9a78563412) << 64 | wide::UInt256(0xf0debc9a78563412));
}

} // namespace doris
Binary file added ...es/docker-compose/hive/scripts/preinstalled_data/hdfs_tvf/test_parquet_decimal256.parquet
View file
Open in desktop
Binary file not shown.
8 changes: 8 additions & 0 deletions regression-test/suites/external_table_p0/tvf/test_hdfs_tvf.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,14 @@ suite("test_hdfs_tvf","external,hive,tvf,external_docker") {
"hadoop.username" = "${hdfsUserName}",
"format" = "${format}") order by s_suppkey limit 20; """

// test parquet decimal256
uri = "${defaultFS}" + "/user/doris/preinstalled_data/hdfs_tvf/test_parquet_decimal256.parquet"
format = "parquet"
qt_parquet_decimal256 """ select * from HDFS(
"uri" = "${uri}",
"hadoop.username" = "${hdfsUserName}",
"format" = "${format}") order by id; """

// test orc
uri = "${defaultFS}" + "/user/doris/preinstalled_data/hdfs_tvf/test_orc.snappy.orc"
format = "orc"
Expand Down
Loading