diff --git a/velox/expression/tests/CastExprTest.cpp b/velox/expression/tests/CastExprTest.cpp index b36d42df86e6..968faf12fc93 100644 --- a/velox/expression/tests/CastExprTest.cpp +++ b/velox/expression/tests/CastExprTest.cpp @@ -75,8 +75,7 @@ class CastExprTest : public functions::test::CastBaseTest { DecimalUtil::kShortDecimalMax, std::nullopt}, DECIMAL(18, 18)); - testComplexCast( - "c0", + testCast( shortFlat, makeNullableFlatVector( {-1, @@ -99,8 +98,7 @@ class CastExprTest : public functions::test::CastBaseTest { HugeInt::build(0xffff, 0xffffffffffffffff), std::nullopt}, DECIMAL(38, 5)); - testComplexCast( - "c0", + testCast( longFlat, makeNullableFlatVector( {-1e33, 0, 1e33, 1.2089258196146293E19, std::nullopt})); @@ -116,8 +114,7 @@ class CastExprTest : public functions::test::CastBaseTest { static_cast(std::numeric_limits::max()) + 1; VELOX_ASSERT_THROW( - testComplexCast( - "c0", + testCast( makeFlatVector({0, tooSmall}, DECIMAL(10, 0)), makeFlatVector(0, 0)), fmt::format( @@ -125,8 +122,7 @@ class CastExprTest : public functions::test::CastBaseTest { TypeTraits::name)); VELOX_ASSERT_THROW( - testComplexCast( - "c0", + testCast( makeFlatVector({0, tooSmall}, DECIMAL(19, 0)), makeFlatVector(0, 0)), fmt::format( @@ -134,8 +130,7 @@ class CastExprTest : public functions::test::CastBaseTest { TypeTraits::name)); VELOX_ASSERT_THROW( - testComplexCast( - "c0", + testCast( makeFlatVector({0, tooBig}, DECIMAL(10, 0)), makeFlatVector(0, 0)), fmt::format( @@ -143,8 +138,7 @@ class CastExprTest : public functions::test::CastBaseTest { TypeTraits::name)); VELOX_ASSERT_THROW( - testComplexCast( - "c0", + testCast( makeFlatVector({0, tooBig}, DECIMAL(19, 0)), makeFlatVector(0, 0)), fmt::format( @@ -161,16 +155,14 @@ class CastExprTest : public functions::test::CastBaseTest { const auto tooBig = static_cast(std::numeric_limits::max()) + 1; - testComplexCast( - "c0", + testCast( makeNullableFlatVector( {0, tooSmall, 0, tooBig, 0, std::nullopt, 0}, DECIMAL(10, 0)), makeNullableFlatVector( {0, std::nullopt, 0, std::nullopt, 0, std::nullopt, 0}), true); - testComplexCast( - "c0", + testCast( makeNullableFlatVector( {0, tooSmall, 0, tooBig, 0, std::nullopt, 0}, DECIMAL(19, 0)), makeNullableFlatVector( @@ -194,8 +186,7 @@ class CastExprTest : public functions::test::CastBaseTest { 7200, std::nullopt}, DECIMAL(6, 2)); - testComplexCast( - "c0", + testCast( shortFlat, makeNullableFlatVector( {-3, @@ -224,8 +215,7 @@ class CastExprTest : public functions::test::CastBaseTest { 720'000'000'000, std::nullopt}, DECIMAL(20, 10)); - testComplexCast( - "c0", + testCast( longFlat, makeNullableFlatVector( {-3, @@ -246,15 +236,13 @@ class CastExprTest : public functions::test::CastBaseTest { void testIntToDecimalCasts() { // integer to short decimal auto input = makeFlatVector({-3, -2, -1, 0, 55, 69, 72}); - testComplexCast( - "c0", + testCast( input, makeFlatVector( {-300, -200, -100, 0, 5'500, 6'900, 7'200}, DECIMAL(6, 2))); // integer to long decimal - testComplexCast( - "c0", + testCast( input, makeFlatVector( {-30'000'000'000, @@ -269,8 +257,7 @@ class CastExprTest : public functions::test::CastBaseTest { // Expected failures: allowed # of integers (precision - scale) in the // target VELOX_ASSERT_THROW( - testComplexCast( - "c0", + testCast( makeFlatVector(std::vector{std::numeric_limits::min()}), makeFlatVector(std::vector{0}, DECIMAL(3, 1))), fmt::format( @@ -278,15 +265,13 @@ class CastExprTest : public functions::test::CastBaseTest { CppToType::name, std::to_string(std::numeric_limits::min()))); VELOX_ASSERT_THROW( - testComplexCast( - "c0", + testCast( makeFlatVector(std::vector{-100}), makeFlatVector(std::vector{0}, DECIMAL(17, 16))), fmt::format( "Cannot cast {} '-100' to DECIMAL(17, 16)", CppToType::name)); VELOX_ASSERT_THROW( - testComplexCast( - "c0", + testCast( makeFlatVector(std::vector{100}), makeFlatVector(std::vector{0}, DECIMAL(17, 16))), fmt::format( @@ -1083,7 +1068,7 @@ TEST_F(CastExprTest, mapCast) { auto expectedMap = makeMapVector( kVectorSize, sizeAt, keyAt, valueAt, nullEvery(3)); - testComplexCast("c0", inputMap, expectedMap); + testCast(inputMap, expectedMap); } // Cast map -> map. @@ -1095,7 +1080,7 @@ TEST_F(CastExprTest, mapCast) { auto expectedMap = makeMapVector( kVectorSize, sizeAt, keyAt, valueAtString, nullEvery(3)); - testComplexCast("c0", inputMap, expectedMap); + testCast(inputMap, expectedMap); } // Cast map -> map. @@ -1107,7 +1092,7 @@ TEST_F(CastExprTest, mapCast) { auto expectedMap = makeMapVector( kVectorSize, sizeAt, keyAtString, valueAt, nullEvery(3)); - testComplexCast("c0", inputMap, expectedMap); + testCast(inputMap, expectedMap); } // null values @@ -1118,14 +1103,13 @@ TEST_F(CastExprTest, mapCast) { auto expectedMap = makeMapVector( kVectorSize, sizeAt, keyAt, valueAt, nullEvery(3), nullEvery(7)); - testComplexCast("c0", inputWithNullValues, expectedMap); + testCast(inputWithNullValues, expectedMap); } // Nulls in result keys are not allowed. { VELOX_ASSERT_THROW( - testComplexCast( - "c0", + testCast( inputMap, makeMapVector( kVectorSize, @@ -1137,8 +1121,7 @@ TEST_F(CastExprTest, mapCast) { false), "Cannot cast BIGINT '0' to TIMESTAMP. Conversion to Timestamp is not supported"); - testComplexCast( - "c0", + testCast( inputMap, makeMapVector( kVectorSize, @@ -1227,7 +1210,7 @@ TEST_F(CastExprTest, arrayCast) { { auto expected = makeArrayVector(kVectorSize, sizeAt, valueAt, nullEvery(3)); - testComplexCast("c0", arrayVector, expected); + testCast(arrayVector, expected); } // Cast array -> array. @@ -1239,7 +1222,7 @@ TEST_F(CastExprTest, arrayCast) { }; auto expected = makeArrayVector( kVectorSize, sizeAt, valueAtString, nullEvery(3)); - testComplexCast("c0", arrayVector, expected); + testCast(arrayVector, expected); } // Make sure that the output of array cast has valid(copyable) data even for @@ -1303,7 +1286,7 @@ TEST_F(CastExprTest, arrayCast) { {{{{std::nullopt, 4}}}}, // row1 }); - testComplexCast("c0", data, expected, true); + testCast(data, expected, true); } } @@ -1327,7 +1310,7 @@ TEST_F(CastExprTest, rowCast) { { auto expectedRowVector = makeRowVector( {doubleVectorNullEvery11, intVectorNullEvery3}, nullEvery(5)); - testComplexCast("c0", rowVector, expectedRowVector); + testCast(rowVector, expectedRowVector); } // Position-based cast: ROW(c0: bigint, c1: double) -> ROW(a: double, b: // bigint) @@ -1336,13 +1319,13 @@ TEST_F(CastExprTest, rowCast) { {"a", "b"}, {doubleVectorNullEvery11, intVectorNullEvery3}, nullEvery(5)); - testComplexCast("c0", rowVector, expectedRowVector); + testCast(rowVector, expectedRowVector); } // Position-based cast: ROW(c0: bigint, c1: double) -> ROW(c0: double) { auto expectedRowVector = makeRowVector({doubleVectorNullEvery11}, nullEvery(5)); - testComplexCast("c0", rowVector, expectedRowVector); + testCast(rowVector, expectedRowVector); } // Name-based cast: ROW(c0: bigint, c1: double) -> ROW(c0: double) dropping @@ -1353,7 +1336,7 @@ TEST_F(CastExprTest, rowCast) { kVectorSize, valueAtInt, [](vector_size_t /* row */) { return true; }); auto expectedRowVector = makeRowVector( {"c0", "b"}, {doubleVectorNullEvery11, intVectorNullAll}, nullEvery(5)); - testComplexCast("c0", rowVector, expectedRowVector); + testCast(rowVector, expectedRowVector); } // Error handling. @@ -1366,7 +1349,7 @@ TEST_F(CastExprTest, rowCast) { {makeFlatVector({1, 2}), makeFlatVector({2, 3})}); expected->setNull(1, true); - testComplexCast("c0", data, expected, true); + testCast(data, expected, true); } { @@ -1393,9 +1376,9 @@ TEST_F(CastExprTest, rowCast) { expected3->setNull(0, true); expected3->setNull(1, true); - testComplexCast("c0", data, expected1, true); - testComplexCast("c0", data, expected2, true); - testComplexCast("c0", data, expected3, true); + testCast(data, expected1, true); + testCast(data, expected2, true); + testCast(data, expected3, true); } // Null handling for nested structs. @@ -1405,7 +1388,7 @@ TEST_F(CastExprTest, rowCast) { auto expected = makeRowVector({makeRowVector({makeFlatVector({1, 0})})}); expected->setNull(1, true); - testComplexCast("c0", data, expected, true); + testCast(data, expected, true); } } @@ -1431,10 +1414,10 @@ TEST_F(CastExprTest, testNullOnFailure) { {1, 2, std::nullopt, std::nullopt, std::nullopt}); // nullOnFailure is true, so we should return null instead of throwing. - testComplexCast("c0", input, expected, true); + testCast(input, expected, true); // nullOnFailure is false, so we should throw. - EXPECT_THROW(testComplexCast("c0", input, expected, false), VeloxUserError); + EXPECT_THROW(testCast(input, expected, false), VeloxUserError); } TEST_F(CastExprTest, toString) { @@ -1474,20 +1457,17 @@ TEST_F(CastExprTest, decimalToFloat) { TEST_F(CastExprTest, decimalToBool) { auto shortFlat = makeNullableFlatVector( {DecimalUtil::kShortDecimalMin, 0, std::nullopt}, DECIMAL(18, 18)); - testComplexCast( - "c0", shortFlat, makeNullableFlatVector({1, 0, std::nullopt})); + testCast(shortFlat, makeNullableFlatVector({1, 0, std::nullopt})); auto longFlat = makeNullableFlatVector( {DecimalUtil::kLongDecimalMin, 0, std::nullopt}, DECIMAL(38, 5)); - testComplexCast( - "c0", longFlat, makeNullableFlatVector({1, 0, std::nullopt})); + testCast(longFlat, makeNullableFlatVector({1, 0, std::nullopt})); } TEST_F(CastExprTest, decimalToVarchar) { auto flatForInline = makeNullableFlatVector( {123456789, -333333333, 0, 5, -9, std::nullopt}, DECIMAL(9, 2)); - testComplexCast( - "c0", + testCast( flatForInline, makeNullableFlatVector( {"1234567.89", @@ -1498,8 +1478,7 @@ TEST_F(CastExprTest, decimalToVarchar) { std::nullopt})); auto shortFlatForZero = makeNullableFlatVector({0}, DECIMAL(6, 0)); - testComplexCast( - "c0", shortFlatForZero, makeNullableFlatVector({"0"})); + testCast(shortFlatForZero, makeNullableFlatVector({"0"})); auto shortFlat = makeNullableFlatVector( {DecimalUtil::kShortDecimalMin, @@ -1509,8 +1488,7 @@ TEST_F(CastExprTest, decimalToVarchar) { DecimalUtil::kShortDecimalMax, std::nullopt}, DECIMAL(18, 18)); - testComplexCast( - "c0", + testCast( shortFlat, makeNullableFlatVector( {"-0.999999999999999999", @@ -1528,8 +1506,7 @@ TEST_F(CastExprTest, decimalToVarchar) { HugeInt::build(0xffff, 0xffffffffffffffff), std::nullopt}, DECIMAL(38, 5)); - testComplexCast( - "c0", + testCast( longFlat, makeNullableFlatVector( {"-999999999999999999999999999999999.99999", @@ -1540,58 +1517,48 @@ TEST_F(CastExprTest, decimalToVarchar) { std::nullopt})); auto longFlatForZero = makeNullableFlatVector({0}, DECIMAL(25, 0)); - testComplexCast( - "c0", longFlatForZero, makeNullableFlatVector({"0"})); + testCast(longFlatForZero, makeNullableFlatVector({"0"})); } TEST_F(CastExprTest, decimalToDecimal) { // short to short, scale up. auto shortFlat = makeFlatVector({-3, -2, -1, 0, 55, 69, 72}, DECIMAL(2, 2)); - testComplexCast( - "c0", + testCast( shortFlat, makeFlatVector( {-300, -200, -100, 0, 5'500, 6'900, 7'200}, DECIMAL(4, 4))); // short to short, scale down. - testComplexCast( - "c0", - shortFlat, - makeFlatVector({0, 0, 0, 0, 6, 7, 7}, DECIMAL(4, 1))); + testCast( + shortFlat, makeFlatVector({0, 0, 0, 0, 6, 7, 7}, DECIMAL(4, 1))); // long to short, scale up. auto longFlat = makeFlatVector({-201, -109, 0, 105, 208}, DECIMAL(20, 2)); - testComplexCast( - "c0", + testCast( longFlat, makeFlatVector( {-201'000, -109'000, 0, 105'000, 208'000}, DECIMAL(10, 5))); // long to short, scale down. - testComplexCast( - "c0", - longFlat, - makeFlatVector({-20, -11, 0, 11, 21}, DECIMAL(10, 1))); + testCast( + longFlat, makeFlatVector({-20, -11, 0, 11, 21}, DECIMAL(10, 1))); // long to long, scale up. - testComplexCast( - "c0", + testCast( longFlat, makeFlatVector( {-20'100'000'000, -10'900'000'000, 0, 10'500'000'000, 20'800'000'000}, DECIMAL(20, 10))); // long to long, scale down. - testComplexCast( - "c0", + testCast( longFlat, makeFlatVector({-20, -11, 0, 11, 21}, DECIMAL(20, 1))); // short to long, scale up. - testComplexCast( - "c0", + testCast( shortFlat, makeFlatVector( {-3'000'000'000, @@ -1604,8 +1571,7 @@ TEST_F(CastExprTest, decimalToDecimal) { DECIMAL(20, 11))); // short to long, scale down. - testComplexCast( - "c0", + testCast( makeFlatVector({-20'500, -190, 12'345, 19'999}, DECIMAL(6, 4)), makeFlatVector({-21, 0, 12, 20}, DECIMAL(20, 1))); @@ -1617,15 +1583,14 @@ TEST_F(CastExprTest, decimalToDecimal) { // Throws exception if CAST fails. VELOX_ASSERT_THROW( - testComplexCast("c0", longFlat, expectedShort), + testCast(longFlat, expectedShort), "Cannot cast DECIMAL '-1000.000' to DECIMAL(6, 4)"); // nullOnFailure is true. - testComplexCast("c0", longFlat, expectedShort, true); + testCast(longFlat, expectedShort, true); // long to short, big numbers. - testComplexCast( - "c0", + testCast( makeNullableFlatVector( {HugeInt::build(-2, 200), HugeInt::build(-1, 300), @@ -1646,15 +1611,13 @@ TEST_F(CastExprTest, decimalToDecimal) { // Overflow case. VELOX_ASSERT_THROW( - testComplexCast( - "c0", + testCast( makeNullableFlatVector( {DecimalUtil::kLongDecimalMax}, DECIMAL(38, 0)), makeNullableFlatVector({0}, DECIMAL(38, 1))), "Cannot cast DECIMAL '99999999999999999999999999999999999999' to DECIMAL(38, 1)"); VELOX_ASSERT_THROW( - testComplexCast( - "c0", + testCast( makeNullableFlatVector( {DecimalUtil::kLongDecimalMin}, DECIMAL(38, 0)), makeNullableFlatVector({0}, DECIMAL(38, 1))), @@ -1672,14 +1635,12 @@ TEST_F(CastExprTest, boolToDecimal) { // Bool to short decimal. auto input = makeFlatVector({true, false, false, true, true, true, false}); - testComplexCast( - "c0", + testCast( input, makeFlatVector({100, 0, 0, 100, 100, 100, 0}, DECIMAL(6, 2))); // Bool to long decimal. - testComplexCast( - "c0", + testCast( input, makeFlatVector( {10'000'000'000, @@ -1693,8 +1654,7 @@ TEST_F(CastExprTest, boolToDecimal) { } TEST_F(CastExprTest, varcharToDecimal) { - testComplexCast( - "c0", + testCast( makeFlatVector( {"9999999999.99", "15", @@ -1756,8 +1716,7 @@ TEST_F(CastExprTest, varcharToDecimal) { DECIMAL(12, 2))); // Truncates the fractional digits with exponent. - testComplexCast( - "c0", + testCast( makeFlatVector( {"112345612.23e-6", "112345662.23e-6", @@ -1782,8 +1741,7 @@ TEST_F(CastExprTest, varcharToDecimal) { const auto minDecimalStr = '-' + std::string(36, '9') + '.' + "99"; const auto maxDecimalStr = std::string(36, '9') + '.' + "99"; - testComplexCast( - "c0", + testCast( makeFlatVector( {StringView(minDecimalStr), StringView(maxDecimalStr), @@ -1801,8 +1759,7 @@ TEST_F(CastExprTest, varcharToDecimal) { const std::string fractionLargeExp = "1.9" + std::string(67, '9') + "e2"; const std::string fractionLargeNegExp = "1000.9" + std::string(67, '9') + "e-2"; - testComplexCast( - "c0", + testCast( makeFlatVector( {StringView(('-' + std::string(38, '9')).data()), StringView(std::string(38, '9').data()), @@ -1820,8 +1777,7 @@ TEST_F(CastExprTest, varcharToDecimal) { const std::string fractionRoundDown = "0." + std::string(38, '9') + "2"; const std::string fractionRoundDownExp = "99." + std::string(36, '9') + "2e-2"; - testComplexCast( - "c0", + testCast( makeFlatVector( {StringView(fractionRoundDown), StringView(fractionRoundDownExp)}), makeConstant(DecimalUtil::kLongDecimalMax, 2, DECIMAL(38, 38))); diff --git a/velox/functions/prestosql/tests/CastBaseTest.h b/velox/functions/prestosql/tests/CastBaseTest.h index d150784febb6..8870d0208264 100644 --- a/velox/functions/prestosql/tests/CastBaseTest.h +++ b/velox/functions/prestosql/tests/CastBaseTest.h @@ -83,6 +83,10 @@ class CastBaseTest : public FunctionBaseTest { bool isTryCast) { core::TypedExprPtr inputField = std::make_shared(fromType, "c0"); + + // It is not sufficient to wrap input in a dictionary as it will be peeled + // off before calling "cast". Apply testing_dictionary function to input to + // ensure that "cast" receives dictionary input. core::TypedExprPtr callExpr = std::make_shared( fromType, std::vector{inputField}, @@ -188,39 +192,79 @@ class CastBaseTest : public FunctionBaseTest { } void testCast( - const TypePtr& fromType, - const TypePtr& toType, const VectorPtr& input, - const VectorPtr& expected) { + const VectorPtr& expected, + std::optional isTryCast = std::nullopt) { + const auto& fromType = input->type(); + const auto& toType = expected->type(); SCOPED_TRACE(fmt::format( "Cast from {} to {}", fromType->toString(), toType->toString())); + auto copy = createCopy(input); // Test with flat encoding. { SCOPED_TRACE("Flat encoding"); - evaluateAndVerify(fromType, toType, makeRowVector({input}), expected); - evaluateAndVerify( - fromType, toType, makeRowVector({input}), expected, true); + if (isTryCast.has_value()) { + evaluateAndVerify( + fromType, + toType, + makeRowVector({input}), + expected, + isTryCast.value()); + } else { + evaluateAndVerify(fromType, toType, makeRowVector({input}), expected); + evaluateAndVerify( + fromType, toType, makeRowVector({input}), expected, true); + } + + // Make sure the input vector does not change. + assertEqualVectors(input, copy); } // Test with constant encoding that repeats the first element five times. { SCOPED_TRACE("Constant encoding"); auto constInput = BaseVector::wrapInConstant(5, 0, input); + auto constantRow = makeRowVector({constInput}); + auto localCopy = createCopy(constantRow); auto constExpected = BaseVector::wrapInConstant(5, 0, expected); - evaluateAndVerify( - fromType, toType, makeRowVector({constInput}), constExpected); - evaluateAndVerify( - fromType, toType, makeRowVector({constInput}), constExpected, true); + if (isTryCast.has_value()) { + evaluateAndVerify( + fromType, + toType, + makeRowVector({constInput}), + constExpected, + isTryCast.value()); + } else { + evaluateAndVerify(fromType, toType, constantRow, constExpected); + evaluateAndVerify( + fromType, toType, makeRowVector({constInput}), constExpected, true); + } + + // Make sure the input vector does not change. + assertEqualVectors(constantRow, localCopy); + assertEqualVectors(input, copy); } // Test with dictionary encoding that reverses the indices. { SCOPED_TRACE("Dictionary encoding"); - evaluateAndVerifyDictEncoding( - fromType, toType, makeRowVector({input}), expected); - evaluateAndVerifyDictEncoding( - fromType, toType, makeRowVector({input}), expected, true); + if (isTryCast.has_value()) { + evaluateAndVerifyDictEncoding( + fromType, + toType, + makeRowVector({input}), + expected, + isTryCast.value()); + } else { + evaluateAndVerifyDictEncoding( + fromType, toType, makeRowVector({input}), expected); + evaluateAndVerifyDictEncoding( + fromType, toType, makeRowVector({input}), expected, true); + } + + // Make sure the input vector does not change. + assertEqualVectors(input, copy); } } @@ -233,7 +277,7 @@ class CastBaseTest : public FunctionBaseTest { auto inputVector = makeNullableFlatVector(input, fromType); auto expectedVector = makeNullableFlatVector(expected, toType); - testCast(fromType, toType, inputVector, expectedVector); + testCast(inputVector, expectedVector); } template @@ -250,72 +294,6 @@ class CastBaseTest : public FunctionBaseTest { expectedErrorMessage); } - void testComplexCast( - const std::string& fromExpression, - const VectorPtr& data, - const VectorPtr& expected, - bool nullOnFailure = false) { - auto rowVector = makeRowVector({data}); - auto rowType = asRowType(rowVector->type()); - auto castExpr = makeCastExpr( - makeTypedExpr(fromExpression, rowType), - expected->type(), - nullOnFailure); - exec::ExprSet exprSet({castExpr}, &execCtx_); - auto copy = createCopy(data); - const auto size = data->size(); - SelectivityVector rows(size); - std::vector result(1); - { - exec::EvalCtx evalCtx(&execCtx_, &exprSet, rowVector.get()); - exprSet.eval(rows, evalCtx, result); - - assertEqualVectors(expected, result[0]); - - // Make sure the input vector does not change. - assertEqualVectors(data, copy); - } - - // Test constant input. - { - // Use last element for constant. - const auto index = size - 1; - auto constantData = BaseVector::wrapInConstant(size, index, data); - auto constantRow = makeRowVector({constantData}); - auto localCopy = createCopy(constantRow); - exec::EvalCtx evalCtx(&execCtx_, &exprSet, constantRow.get()); - exprSet.eval(rows, evalCtx, result); - - // Make sure the input vector does not change. - assertEqualVectors(constantRow, localCopy); - assertEqualVectors(data, copy); - - assertEqualVectors( - BaseVector::wrapInConstant(size, index, expected), result[0]); - } - - // Test dictionary input. It is not sufficient to wrap input in a dictionary - // as it will be peeled off before calling "cast". Apply - // testing_dictionary function to input to ensure that "cast" receives - // dictionary input. - { - auto dictionaryCastExpr = makeCastExpr( - makeTypedExpr( - fmt::format("testing_dictionary({})", fromExpression), rowType), - expected->type(), - nullOnFailure); - exec::ExprSet dictionaryExprSet({dictionaryCastExpr}, &execCtx_); - exec::EvalCtx evalCtx(&execCtx_, &dictionaryExprSet, rowVector.get()); - dictionaryExprSet.eval(rows, evalCtx, result); - - // Make sure the input vector does not change. - assertEqualVectors(data, copy); - - auto indices = functions::test::makeIndicesInReverse(size, pool()); - assertEqualVectors(wrapInDictionary(indices, size, expected), result[0]); - } - } - VectorPtr createCopy(const VectorPtr& input) { VectorPtr result; SelectivityVector rows(input->size()); diff --git a/velox/functions/prestosql/tests/JsonCastTest.cpp b/velox/functions/prestosql/tests/JsonCastTest.cpp index b3305e154c03..6b9ebdd03df4 100644 --- a/velox/functions/prestosql/tests/JsonCastTest.cpp +++ b/velox/functions/prestosql/tests/JsonCastTest.cpp @@ -53,7 +53,7 @@ class JsonCastTest : public functions::test::CastBaseTest { auto expectedVector = makeNullableFlatVector(expected, JSON()); - testCast(fromType, JSON(), arrayVector, expectedVector); + testCast(arrayVector, expectedVector); } template @@ -65,7 +65,7 @@ class JsonCastTest : public functions::test::CastBaseTest { auto expectedVector = makeNullableFlatVector(expected, JSON()); - testCast(fromType, JSON(), mapVector, expectedVector); + testCast(mapVector, expectedVector); } template @@ -86,7 +86,7 @@ class JsonCastTest : public functions::test::CastBaseTest { auto expectedVector = makeNullableFlatVector(expected, JSON()); - testCast(fromType, JSON(), rowVector, expectedVector); + testCast(rowVector, expectedVector); } // Populates offsets and sizes buffers for making array and map vectors. @@ -337,8 +337,8 @@ TEST_F(JsonCastTest, toArrayOfJson) { }, JSON()); - testCast(ARRAY(ARRAY(BIGINT())), ARRAY(JSON()), from, to); - testCast(ARRAY(JSON()), ARRAY(ARRAY(BIGINT())), to, from); + testCast(from, to); + testCast(to, from); } TEST_F(JsonCastTest, fromArray) { @@ -359,22 +359,14 @@ TEST_F(JsonCastTest, fromArray) { ARRAY(UNKNOWN())); auto arrayOfUnknownElementsExpected = makeNullableFlatVector( {"[null,null]", "[null,null]"}, JSON()); - testCast( - ARRAY(UNKNOWN()), - JSON(), - arrayOfUnknownElements, - arrayOfUnknownElementsExpected); + testCast(arrayOfUnknownElements, arrayOfUnknownElementsExpected); // Tests array whose elements are wrapped in a dictionary. auto arrayOfDictElements = makeArrayWithDictionaryElements({1, -2, 3, -4, 5, -6, 7}, 2); auto arrayOfDictElementsExpected = makeNullableFlatVector( {"[null,-6]", "[5,-4]", "[3,-2]", "[1]"}, JSON()); - testCast( - ARRAY(BIGINT()), - JSON(), - arrayOfDictElements, - arrayOfDictElementsExpected); + testCast(arrayOfDictElements, arrayOfDictElementsExpected); // Tests array whose elements are json and wrapped in a dictionary. auto jsonArrayOfDictElements = @@ -384,18 +376,14 @@ TEST_F(JsonCastTest, fromArray) { ARRAY(JSON())); auto jsonArrayOfDictElementsExpected = makeNullableFlatVector( {"[null,f]", "[e,d]", "[c,b]", "[a]"}, JSON()); - testCast( - ARRAY(JSON()), - JSON(), - jsonArrayOfDictElements, - jsonArrayOfDictElementsExpected); + testCast(jsonArrayOfDictElements, jsonArrayOfDictElementsExpected); // Tests array vector with nulls at all rows. auto allNullArray = makeAllNullArrayVector(5, BIGINT()); auto allNullExpected = makeNullableFlatVector( {std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt}, JSON()); - testCast(ARRAY(BIGINT()), JSON(), allNullArray, allNullExpected); + testCast(allNullArray, allNullExpected); } TEST_F(JsonCastTest, fromAllNullOrEmptyArrayOfRows) { @@ -406,7 +394,7 @@ TEST_F(JsonCastTest, fromAllNullOrEmptyArrayOfRows) { auto expected = makeNullableFlatVector( {std::nullopt, "[]", std::nullopt, "[]"}, JSON()); - testCast(data->type(), JSON(), data, expected); + testCast(data, expected); } TEST_F(JsonCastTest, fromAllNullOrEmptyMapOfRows) { @@ -418,7 +406,7 @@ TEST_F(JsonCastTest, fromAllNullOrEmptyMapOfRows) { auto expected = makeNullableFlatVector( {std::nullopt, "{}", std::nullopt, "{}"}, JSON()); - testCast(data->type(), JSON(), data, expected); + testCast(data, expected); } TEST_F(JsonCastTest, fromMap) { @@ -474,11 +462,7 @@ TEST_F(JsonCastTest, fromMap) { R"({"a":null})"}, JSON()); - testCast( - MAP(VARCHAR(), UNKNOWN()), - JSON(), - mapOfUnknownValues, - mapOfUnknownValuesExpected); + testCast(mapOfUnknownValues, mapOfUnknownValuesExpected); // Tests map whose elements are wrapped in a dictionary. std::vector> values{ @@ -491,11 +475,7 @@ TEST_F(JsonCastTest, fromMap) { R"({"b":2.2,"c":3.14})", R"({"a":1100})"}, JSON()); - testCast( - MAP(VARCHAR(), DOUBLE()), - JSON(), - mapOfDictElements, - mapOfDictElementsExpected); + testCast(mapOfDictElements, mapOfDictElementsExpected); // Tests map whose elements are json and wrapped in a dictionary. auto jsonMapOfDictElements = @@ -503,18 +483,14 @@ TEST_F(JsonCastTest, fromMap) { auto jsonMapOfDictElementsExpected = makeNullableFlatVector( {"{f:-6E-10,g:null}", "{d:-4.4,e:null}", "{b:2.2,c:3.14}", "{a:1100}"}, JSON()); - testCast( - MAP(JSON(), DOUBLE()), - JSON(), - jsonMapOfDictElements, - jsonMapOfDictElementsExpected); + testCast(jsonMapOfDictElements, jsonMapOfDictElementsExpected); // Tests map vector with nulls at all rows. auto allNullMap = makeAllNullMapVector(5, VARCHAR(), BIGINT()); auto allNullExpected = makeNullableFlatVector( {std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt}, JSON()); - testCast(MAP(VARCHAR(), BIGINT()), JSON(), allNullMap, allNullExpected); + testCast(allNullMap, allNullExpected); } TEST_F(JsonCastTest, fromRow) { @@ -554,22 +530,14 @@ TEST_F(JsonCastTest, fromRow) { auto rowOfUnknownChildrenExpected = makeNullableFlatVector( {"[null,null]", "[null,null]"}, JSON()); - testCast( - ROW({UNKNOWN(), UNKNOWN()}), - JSON(), - rowOfUnknownChildren, - rowOfUnknownChildrenExpected); + testCast(rowOfUnknownChildren, rowOfUnknownChildrenExpected); // Tests row whose children are wrapped in dictionaries. auto rowOfDictElements = makeRowWithDictionaryElements( {{1, 2, 3}, {4, 5, 6}, {7, 8, 9}}, ROW({BIGINT(), BIGINT(), BIGINT()})); auto rowOfDictElementsExpected = makeNullableFlatVector( {"[null,null,null]", "[2,5,8]", "[1,4,7]"}, JSON()); - testCast( - ROW({BIGINT(), BIGINT(), BIGINT()}), - JSON(), - rowOfDictElements, - rowOfDictElementsExpected); + testCast(rowOfDictElements, rowOfDictElementsExpected); // Tests row whose children are json and wrapped in dictionaries. auto jsonRowOfDictElements = makeRowWithDictionaryElements( @@ -579,11 +547,7 @@ TEST_F(JsonCastTest, fromRow) { ROW({JSON(), JSON(), JSON()})); auto jsonRowOfDictElementsExpected = makeNullableFlatVector( {"[null,null,null]", "[a2,b2,c2]", "[a1,b1,c1]"}, JSON()); - testCast( - ROW({JSON(), JSON(), JSON()}), - JSON(), - jsonRowOfDictElements, - jsonRowOfDictElementsExpected); + testCast(jsonRowOfDictElements, jsonRowOfDictElementsExpected); // Tests row vector with nulls at all rows. auto allNullChild = makeAllNullFlatVector(5); @@ -594,7 +558,7 @@ TEST_F(JsonCastTest, fromRow) { auto allNullExpected = makeNullableFlatVector( {std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt}, JSON()); - testCast(ROW({BIGINT()}), JSON(), allNullRow, allNullExpected); + testCast(allNullRow, allNullExpected); } TEST_F(JsonCastTest, fromNested) { @@ -645,11 +609,7 @@ TEST_F(JsonCastTest, fromNested) { auto expectedVector = makeNullableFlatVector(expected, JSON()); - testCast( - ROW({MAP(VARCHAR(), ARRAY(BIGINT())), ARRAY(MAP(VARCHAR(), BIGINT()))}), - JSON(), - rowVector, - expectedVector); + testCast(rowVector, expectedVector); } TEST_F(JsonCastTest, unsupportedTypes) { @@ -934,19 +894,20 @@ TEST_F(JsonCastTest, toArray) { {{}}, std::nullopt}); - testCast(JSON(), ARRAY(VARCHAR()), data, expected); + testCast(data, expected); // Tests array that has null at every row. data = makeNullableFlatVector( - {"null"_sv, "null"_sv, "null"_sv, "null"_sv, std::nullopt}); + {"null"_sv, "null"_sv, "null"_sv, "null"_sv, std::nullopt}, JSON()); expected = makeNullableArrayVector( {std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt}); - testCast(JSON(), ARRAY(BIGINT()), data, expected); + testCast(data, expected); - data = makeNullableFlatVector({"[233897314173811950000]"_sv}); + data = makeNullableFlatVector( + {"[233897314173811950000]"_sv}, JSON()); expected = makeArrayVector({{233897314173811950000.0}}); - testCast(JSON(), ARRAY(DOUBLE()), data, expected); + testCast(data, expected); } TEST_F(JsonCastTest, toMap) { @@ -962,7 +923,7 @@ TEST_F(JsonCastTest, toMap) { {{}}, std::nullopt}); - testCast(JSON(), MAP(VARCHAR(), VARCHAR()), data, expected); + testCast(data, expected); // Tests map of non-string keys. data = makeNullableFlatVector( @@ -977,15 +938,15 @@ TEST_F(JsonCastTest, toMap) { {{}}, std::nullopt}); - testCast(JSON(), MAP(BIGINT(), DOUBLE()), data, expected); + testCast(data, expected); // Tests map that has null at every row. data = makeNullableFlatVector( - {"null"_sv, "null"_sv, "null"_sv, "null"_sv, std::nullopt}); + {"null"_sv, "null"_sv, "null"_sv, "null"_sv, std::nullopt}, JSON()); expected = makeNullableMapVector( {std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt}); - testCast(JSON(), MAP(VARCHAR(), BIGINT()), data, expected); + testCast(data, expected); // Null keys or non-string keys in JSON maps are not allowed. testThrow( @@ -1015,7 +976,7 @@ TEST_F(JsonCastTest, orderOfKeys) { }, MAP(VARCHAR(), JSON())); - testCast(JSON(), MAP(VARCHAR(), JSON()), data, map); + testCast(data, map); } TEST_F(JsonCastTest, toRow) { @@ -1032,20 +993,16 @@ TEST_F(JsonCastTest, toRow) { auto child3 = makeNullableFlatVector({true, false, std::nullopt, std::nullopt}); - testCast( - JSON(), - ROW({BIGINT(), VARCHAR(), BOOLEAN()}), - array, - makeRowVector({child1, child2, child3})); + testCast(array, makeRowVector({child1, child2, child3})); // Test casting to ROW from JSON objects. auto map = makeNullableFlatVector( - {R"({"k1":123,"k2":"abc","k3":true})"_sv, - R"({"k2":"abc","k3":true,"k1":123})"_sv, - R"({"k1":123,"k3":true,"k1":456})"_sv, - R"({"k4":123,"k5":"abc","k3":false})"_sv, - R"({"k1":null,"k3":false})"_sv, - R"({"k1":null,"k3":null,"k2":null})"_sv}, + {R"({"c0":123,"c1":"abc","c2":true})"_sv, + R"({"c1":"abc","c2":true,"c0":123})"_sv, + R"({"c0":123,"c2":true,"c0":456})"_sv, + R"({"c3":123,"c4":"abc","c2":false})"_sv, + R"({"c0":null,"c2":false})"_sv, + R"({"c0":null,"c2":null,"c1":null})"_sv}, JSON()); auto child4 = makeNullableFlatVector( {123, 123, 456, std::nullopt, std::nullopt, std::nullopt}); @@ -1059,40 +1016,28 @@ TEST_F(JsonCastTest, toRow) { auto child6 = makeNullableFlatVector( {true, true, true, false, false, std::nullopt}); - testCast( - JSON(), - ROW({"k1", "k2", "k3"}, {BIGINT(), VARCHAR(), BOOLEAN()}), - map, - makeRowVector({child4, child5, child6})); + testCast(map, makeRowVector({child4, child5, child6})); // Use a mix of lower case and upper case JSON keys. map = makeNullableFlatVector( - {R"({"K1":123,"k2":"abc","k3":true})"_sv, - R"({"K2":"abc","k3":true,"k1":123})"_sv, - R"({"k1":123,"k3":true,"K1":456})"_sv, - R"({"k4":123,"K5":"abc","k3":false})"_sv, - R"({"k1":null,"K3":false})"_sv, - R"({"k1":null,"k3":null,"K2":null})"_sv}, + {R"({"c0":123,"c1":"abc","c2":true})"_sv, + R"({"c1":"abc","c2":true,"c0":123})"_sv, + R"({"c0":123,"c2":true,"c0":456})"_sv, + R"({"c3":123,"c4":"abc","c2":false})"_sv, + R"({"c0":null,"c2":false})"_sv, + R"({"c0":null,"c2":null,"c1":null})"_sv}, JSON()); - testCast( - JSON(), - ROW({"k1", "k2", "k3"}, {BIGINT(), VARCHAR(), BOOLEAN()}), - map, - makeRowVector({child4, child5, child6})); + testCast(map, makeRowVector({child4, child5, child6})); // Use a mix of lower case and upper case field names in target ROW type. - testCast( - JSON(), - ROW({"K1", "k2", "K3"}, {BIGINT(), VARCHAR(), BOOLEAN()}), - map, - makeRowVector({child4, child5, child6})); + testCast(map, makeRowVector({child4, child5, child6})); // Test casting to ROW from JSON null. - auto null = makeNullableFlatVector({"null"_sv}); + auto null = makeNullableFlatVector({"null"_sv}, JSON()); auto nullExpected = makeRowVector(ROW({BIGINT(), DOUBLE()}), 1); nullExpected->setNull(0, true); - testCast(JSON(), ROW({BIGINT(), DOUBLE()}), null, nullExpected); + testCast(null, nullExpected); } TEST_F(JsonCastTest, toNested) { @@ -1105,10 +1050,11 @@ TEST_F(JsonCastTest, toNested) { {{{{}}}}, {{}}}); - testCast(JSON(), ARRAY(ARRAY(VARCHAR())), array, arrayExpected); + testCast(array, arrayExpected); auto map = makeNullableFlatVector( - {R"({"1":[1.1,1.2],"2":[2,2.1]})"_sv, R"({"3":null,"4":[4.1,4.2]})"_sv}); + {R"({"1":[1.1,1.2],"2":[2,2.1]})"_sv, R"({"3":null,"4":[4.1,4.2]})"_sv}, + JSON()); auto keys = makeNullableFlatVector({"1"_sv, "2"_sv, "3"_sv, "4"_sv}); auto innerArray = makeNullableArrayVector( @@ -1127,7 +1073,7 @@ TEST_F(JsonCastTest, toNested) { sizes, keys, innerArray); - testCast(JSON(), MAP(VARCHAR(), ARRAY(DOUBLE())), map, mapExpected); + testCast(map, mapExpected); } TEST_F(JsonCastTest, toArrayAndMapOfJson) { @@ -1138,7 +1084,7 @@ TEST_F(JsonCastTest, toArrayAndMapOfJson) { {{"[1,2]"_sv, "[null]"_sv, "null"_sv, "\"3\""_sv}, {"[]"_sv}, {}}, ARRAY(JSON())); - testCast(JSON(), ARRAY(JSON()), array, arrayExpected); + testCast(array, arrayExpected); // Test casting to map of JSON values. auto map = makeNullableFlatVector( @@ -1155,7 +1101,7 @@ TEST_F(JsonCastTest, toArrayAndMapOfJson) { {}}, MAP(VARCHAR(), JSON())); - testCast(JSON(), MAP(VARCHAR(), JSON()), map, mapExpected); + testCast(map, mapExpected); // The type of map keys is not allowed to be JSON. testThrow( diff --git a/velox/functions/prestosql/tests/TimestampWithTimeZoneCastTest.cpp b/velox/functions/prestosql/tests/TimestampWithTimeZoneCastTest.cpp index 20be1f68d9e9..3e92c985c0d0 100644 --- a/velox/functions/prestosql/tests/TimestampWithTimeZoneCastTest.cpp +++ b/velox/functions/prestosql/tests/TimestampWithTimeZoneCastTest.cpp @@ -57,7 +57,7 @@ TEST_F(TimestampWithTimeZoneCastTest, fromTimestamp) { makeFlatVector({0, 0, 0})); expected->setNull(1, true); - testCast(TIMESTAMP(), TIMESTAMP_WITH_TIME_ZONE(), tsVector, expected); + testCast(tsVector, expected); } TEST_F(TimestampWithTimeZoneCastTest, toTimestamp) { @@ -71,7 +71,7 @@ TEST_F(TimestampWithTimeZoneCastTest, toTimestamp) { setQueryTimeZone(timezone); auto expected = makeNullableFlatVector( {Timestamp(1996, 0), std::nullopt, Timestamp(19920, 0)}); - testCast(TIMESTAMP_WITH_TIME_ZONE(), TIMESTAMP(), tsWithTZVector, expected); + testCast(tsWithTZVector, expected); // Cast('1969-12-31 16:00:00 -08:00' as timestamp). auto result = evaluateOnce( @@ -84,7 +84,7 @@ TEST_F(TimestampWithTimeZoneCastTest, toTimestamp) { disableAdjustTimestampToTimezone(); auto expected = makeNullableFlatVector( {Timestamp(1996, 0), std::nullopt, Timestamp(19920 - 8 * 3600, 0)}); - testCast(TIMESTAMP_WITH_TIME_ZONE(), TIMESTAMP(), tsWithTZVector, expected); + testCast(tsWithTZVector, expected); // Cast('1969-12-31 16:00:00 -08:00' as timestamp). auto result = evaluateOnce( diff --git a/velox/functions/sparksql/tests/SparkCastExprTest.cpp b/velox/functions/sparksql/tests/SparkCastExprTest.cpp index a09a3a9c780d..dd11315e9b17 100644 --- a/velox/functions/sparksql/tests/SparkCastExprTest.cpp +++ b/velox/functions/sparksql/tests/SparkCastExprTest.cpp @@ -46,8 +46,7 @@ class SparkCastExprTest : public functions::test::CastBaseTest { 7200, std::nullopt}, DECIMAL(6, 2)); - testComplexCast( - "c0", + testCast( shortFlat, makeNullableFlatVector( {-3, @@ -76,8 +75,7 @@ class SparkCastExprTest : public functions::test::CastBaseTest { 720'000'000'000, std::nullopt}, DECIMAL(20, 10)); - testComplexCast( - "c0", + testCast( longFlat, makeNullableFlatVector( {-3, @@ -442,21 +440,17 @@ TEST_F(SparkCastExprTest, overflow) { 7200, std::nullopt}, DECIMAL(5, 1)); - testComplexCast( - "c0", + testCast( shortFlat, makeNullableFlatVector( {-44, -4, 26, 56, -100, 0, 124, 117, 63, -78, -48, std::nullopt})); - testComplexCast( - "c0", + testCast( makeNullableFlatVector({214748364890}, DECIMAL(12, 2)), makeNullableFlatVector({0})); - testComplexCast( - "c0", + testCast( makeNullableFlatVector({214748364890}, DECIMAL(12, 2)), makeNullableFlatVector({-2147483648})); - testComplexCast( - "c0", + testCast( makeNullableFlatVector({214748364890}, DECIMAL(12, 2)), makeNullableFlatVector({2147483648})); } @@ -525,8 +519,7 @@ TEST_F(SparkCastExprTest, fromString) { "timestamp", {"\n\f\r\t\n\u001F 2000-01-01 12:21:56\u000B\u001C\u001D\u001E"}, {Timestamp(946729316, 0)}); - testComplexCast( - "c0", + testCast( makeFlatVector( {" 9999999999.99", "9999999999.99 ",