Skip to content

Commit

Permalink
Fix duckValueAt for DATE values (facebookincubator#7806)
Browse files Browse the repository at this point in the history 
Summary:
Pull Request resolved: facebookincubator#7806

Fix JoinFuzzer's failure when processing DuckDB results of type MAP<DATE,SMALLINT>.

duckValueAt<TypeKind> template needs to be specialized for TypeKind::INTEGER to
allow it to process both INTEGER and DATE values correctly.

Fixes facebookincubator#7744

```
I1130 04:11:53.702302 81607 JoinFuzzer.cpp:867] ==============================> Started iteration 3 (seed: 923335250)
I1130 04:11:53.710649 81607 JoinFuzzer.cpp:281] Executing query plan:
-- HashJoin[LEFT SEMI (PROJECT) t1=u1 AND t3=u3 AND t0=u0 AND t2=u2] -> tp6:MAP<DATE,SMALLINT>, tp4:INTEGER, t2:TINYINT, tp5:INTEGER, t3:DOUBLE, t1:INTEGER, match:BOOLEAN
  -- Values[500 rows in 5 vectors] -> t0:TINYINT, t1:INTEGER, t2:TINYINT, t3:DOUBLE, tp4:INTEGER, tp5:INTEGER, tp6:MAP<DATE,SMALLINT>
  -- Values[55 rows in 5 vectors] -> u0:TINYINT, u1:INTEGER, u2:TINYINT, u3:DOUBLE

I1130 04:11:53.818156 81607 JoinFuzzer.cpp:299] Results: [ROW ROW<tp6:MAP<DATE,SMALLINT>,tp4:INTEGER,t2:TINYINT,tp5:INTEGER,t3:DOUBLE,t1:INTEGER,match:BOOLEAN>: 500 elements, no nulls]
terminate called after throwing an instance of 'duckdb::InvalidInputException'
  what():  Invalid Input Error: Failed to cast value: Unimplemented type for cast (INTEGER -> DATE)

```

Reviewed By: pedroerp

Differential Revision: D51708552

fbshipit-source-id: e8ea9af6e3ed9f10fdb5231751b44b758c189a1b
  • Loading branch information
@mbasmanova @facebook-github-bot
mbasmanova authored and facebook-github-bot committed Nov 30, 2023
1 parent bb193f3 commit 0494224
Show file tree
Hide file tree
Showing 3 changed files with 52 additions and 38 deletions.
61 changes: 31 additions & 30 deletions velox/dwio/common/tests/utils/BatchMaker.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ VectorPtr createScalar(
BufferPtr values = AlignedBuffer::allocate<T>(size, &pool);
auto valuesPtr = values->asMutableRange<T>();

BufferPtr nulls = AlignedBuffer::allocate<char>(bits::nbytes(size), &pool);
BufferPtr nulls = allocateNulls(size, &pool);
auto* nullsPtr = nulls->asMutable<uint64_t>();

size_t nullCount = 0;
Expand All @@ -68,7 +68,7 @@ VectorPtr createScalar(

template <TypeKind KIND>
VectorPtr BatchMaker::createVector(
const std::shared_ptr<const Type>& /* unused */,
const TypePtr& /* unused */,
size_t /* unused */,
memory::MemoryPool& /* unused */,
std::mt19937& /* unused */,
Expand All @@ -78,7 +78,7 @@ VectorPtr BatchMaker::createVector(

template <>
VectorPtr BatchMaker::createVector<TypeKind::BOOLEAN>(
const std::shared_ptr<const Type>& /* unused */,
const TypePtr& /* unused */,
size_t size,
MemoryPool& pool,
std::mt19937& gen,
Expand All @@ -93,7 +93,7 @@ VectorPtr BatchMaker::createVector<TypeKind::BOOLEAN>(

template <>
VectorPtr BatchMaker::createVector<TypeKind::TINYINT>(
const std::shared_ptr<const Type>& /* unused */,
const TypePtr& /* unused */,
size_t size,
MemoryPool& pool,
std::mt19937& gen,
Expand All @@ -108,7 +108,7 @@ VectorPtr BatchMaker::createVector<TypeKind::TINYINT>(

template <>
VectorPtr BatchMaker::createVector<TypeKind::SMALLINT>(
const std::shared_ptr<const Type>& /* unused */,
const TypePtr& /* unused */,
size_t size,
MemoryPool& pool,
std::mt19937& gen,
Expand All @@ -123,7 +123,7 @@ VectorPtr BatchMaker::createVector<TypeKind::SMALLINT>(

template <>
VectorPtr BatchMaker::createVector<TypeKind::INTEGER>(
const std::shared_ptr<const Type>& /* unused */,
const TypePtr& type,
size_t size,
MemoryPool& pool,
std::mt19937& gen,
Expand All @@ -133,12 +133,13 @@ VectorPtr BatchMaker::createVector<TypeKind::INTEGER>(
gen,
[&gen]() { return static_cast<int32_t>(Random::rand32(gen)); },
pool,
isNullAt);
isNullAt,
type);
}

template <>
VectorPtr BatchMaker::createVector<TypeKind::BIGINT>(
const std::shared_ptr<const Type>& /* unused */,
const TypePtr& /*type*/,
size_t size,
MemoryPool& pool,
std::mt19937& gen,
Expand All @@ -149,7 +150,7 @@ VectorPtr BatchMaker::createVector<TypeKind::BIGINT>(

template <>
VectorPtr BatchMaker::createVector<TypeKind::REAL>(
const std::shared_ptr<const Type>& /* unused */,
const TypePtr& /* unused */,
size_t size,
MemoryPool& pool,
std::mt19937& gen,
Expand All @@ -164,7 +165,7 @@ VectorPtr BatchMaker::createVector<TypeKind::REAL>(

template <>
VectorPtr BatchMaker::createVector<TypeKind::DOUBLE>(
const std::shared_ptr<const Type>& /* unused */,
const TypePtr& /* unused */,
size_t size,
MemoryPool& pool,
std::mt19937& gen,
Expand All @@ -179,7 +180,7 @@ VectorPtr BatchMaker::createVector<TypeKind::DOUBLE>(

template <>
VectorPtr BatchMaker::createVector<TypeKind::HUGEINT>(
const std::shared_ptr<const Type>& type,
const TypePtr& type,
size_t size,
MemoryPool& pool,
std::mt19937& gen,
Expand Down Expand Up @@ -249,7 +250,7 @@ VectorPtr createBinary(

template <>
VectorPtr BatchMaker::createVector<TypeKind::VARCHAR>(
const std::shared_ptr<const Type>& /* unused */,
const TypePtr& /* unused */,
size_t size,
MemoryPool& pool,
std::mt19937& gen,
Expand All @@ -259,7 +260,7 @@ VectorPtr BatchMaker::createVector<TypeKind::VARCHAR>(

template <>
VectorPtr BatchMaker::createVector<TypeKind::VARBINARY>(
const std::shared_ptr<const Type>& /* unused */,
const TypePtr& /* unused */,
size_t size,
MemoryPool& pool,
std::mt19937& gen,
Expand All @@ -269,7 +270,7 @@ VectorPtr BatchMaker::createVector<TypeKind::VARBINARY>(

template <>
VectorPtr BatchMaker::createVector<TypeKind::TIMESTAMP>(
const std::shared_ptr<const Type>& /* unused */,
const TypePtr& /* unused */,
size_t size,
MemoryPool& pool,
std::mt19937& gen,
Expand All @@ -289,30 +290,30 @@ VectorPtr BatchMaker::createVector<TypeKind::TIMESTAMP>(

template <>
VectorPtr BatchMaker::createVector<TypeKind::ROW>(
const std::shared_ptr<const Type>& type,
const TypePtr& type,
size_t size,
MemoryPool& pool,
std::mt19937& gen,
std::function<bool(vector_size_t /*index*/)> isNullAt);

template <>
VectorPtr BatchMaker::createVector<TypeKind::ARRAY>(
const std::shared_ptr<const Type>& type,
const TypePtr& type,
size_t size,
MemoryPool& pool,
std::mt19937& gen,
std::function<bool(vector_size_t /*index*/)> isNullAt);

template <>
VectorPtr BatchMaker::createVector<TypeKind::MAP>(
const std::shared_ptr<const Type>& type,
const TypePtr& type,
size_t size,
MemoryPool& pool,
std::mt19937& gen,
std::function<bool(vector_size_t /*index*/)> isNullAt);

VectorPtr createRows(
const std::shared_ptr<const Type>& type,
const TypePtr& type,
size_t size,
bool allowNulls,
MemoryPool& pool,
Expand All @@ -322,7 +323,7 @@ VectorPtr createRows(
size_t nullCount = 0;

if (allowNulls) {
nulls = AlignedBuffer::allocate<char>(bits::nbytes(size), &pool);
nulls = allocateNulls(size, &pool);
auto* nullsPtr = nulls->asMutable<uint64_t>();
for (size_t i = 0; i < size; ++i) {
auto notNull = isNotNull(gen, i, isNullAt);
Expand Down Expand Up @@ -353,7 +354,7 @@ VectorPtr createRows(

template <>
VectorPtr BatchMaker::createVector<TypeKind::ROW>(
const std::shared_ptr<const Type>& type,
const TypePtr& type,
size_t size,
MemoryPool& pool,
std::mt19937& gen,
Expand All @@ -363,18 +364,18 @@ VectorPtr BatchMaker::createVector<TypeKind::ROW>(

template <>
VectorPtr BatchMaker::createVector<TypeKind::ARRAY>(
const std::shared_ptr<const Type>& type,
const TypePtr& type,
size_t size,
MemoryPool& pool,
std::mt19937& gen,
std::function<bool(vector_size_t /*index*/)> isNullAt) {
BufferPtr offsets = AlignedBuffer::allocate<int32_t>(size, &pool);
BufferPtr offsets = allocateOffsets(size, &pool);
auto* offsetsPtr = offsets->asMutable<int32_t>();

BufferPtr lengths = AlignedBuffer::allocate<vector_size_t>(size, &pool);
BufferPtr lengths = allocateSizes(size, &pool);
auto* lengthsPtr = lengths->asMutable<vector_size_t>();

BufferPtr nulls = AlignedBuffer::allocate<char>(bits::nbytes(size), &pool);
BufferPtr nulls = allocateNulls(size, &pool);
auto* nullsPtr = nulls->asMutable<uint64_t>();

size_t nullCount = 0;
Expand Down Expand Up @@ -564,18 +565,18 @@ VectorPtr createMapKeys(

template <>
VectorPtr BatchMaker::createVector<TypeKind::MAP>(
const std::shared_ptr<const Type>& type,
const TypePtr& type,
size_t size,
MemoryPool& pool,
std::mt19937& gen,
std::function<bool(vector_size_t /*index*/)> isNullAt) {
BufferPtr offsets = AlignedBuffer::allocate<vector_size_t>(size, &pool);
BufferPtr offsets = allocateOffsets(size, &pool);
auto* offsetsPtr = offsets->asMutable<vector_size_t>();

BufferPtr lengths = AlignedBuffer::allocate<vector_size_t>(size, &pool);
BufferPtr lengths = allocateSizes(size, &pool);
auto* lengthsPtr = lengths->asMutable<vector_size_t>();

BufferPtr nulls = AlignedBuffer::allocate<char>(bits::nbytes(size), &pool);
BufferPtr nulls = allocateNulls(size, &pool);
auto* nullsPtr = nulls->asMutable<uint64_t>();

size_t nullCount = 0;
Expand Down Expand Up @@ -613,7 +614,7 @@ VectorPtr BatchMaker::createVector<TypeKind::MAP>(
}

VectorPtr BatchMaker::createBatch(
const std::shared_ptr<const Type>& type,
const TypePtr& type,
uint64_t capacity,
MemoryPool& memoryPool,
std::mt19937& gen,
Expand All @@ -625,7 +626,7 @@ VectorPtr BatchMaker::createBatch(
}

VectorPtr BatchMaker::createBatch(
const std::shared_ptr<const Type>& type,
const TypePtr& type,
uint64_t capacity,
MemoryPool& memoryPool,
std::function<bool(vector_size_t /*index*/)> isNullAt,
Expand Down
8 changes: 4 additions & 4 deletions velox/dwio/common/tests/utils/BatchMaker.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,30 +31,30 @@ void propagateNullsRecursive(BaseVector& vector);

struct BatchMaker {
static VectorPtr createBatch(
const std::shared_ptr<const Type>& type,
const TypePtr& type,
uint64_t capacity,
memory::MemoryPool& memoryPool,
std::mt19937& gen,
std::function<bool(vector_size_t /*index*/)> isNullAt = nullptr);

static VectorPtr createBatch(
const std::shared_ptr<const Type>& type,
const TypePtr& type,
uint64_t capacity,
memory::MemoryPool& memoryPool,
std::function<bool(vector_size_t /*index*/)> isNullAt = nullptr,
std::mt19937::result_type seed = std::mt19937::default_seed);

template <TypeKind KIND>
static VectorPtr createVector(
const std::shared_ptr<const Type>& type,
const TypePtr& type,
size_t size,
memory::MemoryPool& pool,
std::mt19937& gen,
std::function<bool(vector_size_t /*index*/)> isNullAt = nullptr);

template <TypeKind KIND>
static VectorPtr createVector(
const std::shared_ptr<const Type>& type,
const TypePtr& type,
size_t size,
memory::MemoryPool& pool,
std::function<bool(vector_size_t /*index*/)> isNullAt = nullptr,
Expand Down
21 changes: 17 additions & 4 deletions velox/exec/tests/utils/QueryAssertions.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,18 @@ ::duckdb::Value duckValueAt<TypeKind::TIMESTAMP>(
veloxTimestampToDuckDB(vector->as<SimpleVector<T>>()->valueAt(index)));
}

template <>
::duckdb::Value duckValueAt<TypeKind::INTEGER>(
const VectorPtr& vector,
vector_size_t index) {
auto type = vector->type();
if (type->isDate()) {
return ::duckdb::Value::DATE(::duckdb::Date::EpochDaysToDate(
vector->as<SimpleVector<int32_t>>()->valueAt(index)));
}
return ::duckdb::Value(vector->as<SimpleVector<int32_t>>()->valueAt(index));
}

template <>
::duckdb::Value duckValueAt<TypeKind::BIGINT>(
const VectorPtr& vector,
Expand Down Expand Up @@ -871,11 +883,12 @@ void DuckDbQueryRunner::createTable(
auto value = ::duckdb::Value::INTERVAL(
0, 0, columnVector->as<SimpleVector<int64_t>>()->valueAt(row));
appender.Append(value);
} else if (type->isDate()) {
auto value = ::duckdb::Value::DATE(::duckdb::Date::EpochDaysToDate(
columnVector->as<SimpleVector<int32_t>>()->valueAt(row)));
appender.Append(value);
} else {
VELOX_CHECK(
type->equivalent(*columnVector->type()),
"{} vs. {}",
type->toString(),
columnVector->toString())
auto value = VELOX_DYNAMIC_SCALAR_TYPE_DISPATCH(
duckValueAt, type->kind(), columnVector, row);
appender.Append(value);
Expand Down

0 comments on commit 0494224

Please sign in to comment.