Skip to content

Commit

Permalink
Support decimal in fuzzer test
Browse files Browse the repository at this point in the history
  • Loading branch information
rui-mo committed Apr 11, 2024
1 parent ae741ca commit 0c1c343
Show file tree
Hide file tree
Showing 8 changed files with 30 additions and 62 deletions.
2 changes: 2 additions & 0 deletions velox/docs/develop/testing/fuzzer.rst
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,8 @@ Below are arguments that toggle certain fuzzer features in Expression Fuzzer:

* ``--velox_fuzzer_enable_complex_types``: Enable testing of function signatures with complex argument or return types. Default is false.

* ``--velox_fuzzer_enable_decimal_type``: Enable testing of function signatures with decimal argument or return type. Default is false.

* ``--lazy_vector_generation_ratio``: Specifies the probability with which columns in the input row vector will be selected to be wrapped in lazy encoding (expressed as double from 0 to 1). Default is 0.0.

* ``--velox_fuzzer_enable_column_reuse``: Enable generation of expressions where one input column can be used by multiple subexpressions. Default is false.
Expand Down
20 changes: 0 additions & 20 deletions velox/expression/ReverseSignatureBinder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,27 +18,7 @@

namespace facebook::velox::exec {

bool ReverseSignatureBinder::hasConstrainedIntegerVariable(
const TypeSignature& type) const {
if (type.parameters().empty()) {
auto it = variables().find(type.baseName());
return it != variables().end() && it->second.isIntegerParameter() &&
it->second.constraint() != "";
}

const auto& parameters = type.parameters();
for (const auto& parameter : parameters) {
if (hasConstrainedIntegerVariable(parameter)) {
return true;
}
}
return false;
}

bool ReverseSignatureBinder::tryBind() {
if (hasConstrainedIntegerVariable(signature_.returnType())) {
return false;
}
tryBindSucceeded_ =
SignatureBinderBase::tryBind(signature_.returnType(), returnType_);
return tryBindSucceeded_;
Expand Down
4 changes: 0 additions & 4 deletions velox/expression/ReverseSignatureBinder.h
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,6 @@ class ReverseSignatureBinder : private SignatureBinderBase {
}

private:
// Return whether there is a constraint on an integer variable in type
// signature.
bool hasConstrainedIntegerVariable(const TypeSignature& type) const;

const TypePtr returnType_;

// True if 'tryBind' has been called and succeeded. False otherwise.
Expand Down
30 changes: 18 additions & 12 deletions velox/expression/tests/ExpressionFuzzer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -624,16 +624,20 @@ bool useTypeName(

bool isSupportedSignature(
const exec::FunctionSignature& signature,
bool enableComplexType) {
// Not supporting lambda functions, or functions using decimal and
// timestamp with time zone types.
bool enableComplexType,
bool enableDecimalType) {
// When enableComplexType is disabled, not supporting complex functions.
const bool useComplexType = useTypeName(signature, "array") ||
useTypeName(signature, "map") || useTypeName(signature, "row");
// When enableDecimalType is disabled, not supporting decimal functions. Not
// supporting lambda functions, or functions using timestamp with time zone
// types.
return !(
useTypeName(signature, "opaque") ||
// useTypeName(signature, "long_decimal") ||
// useTypeName(signature, "short_decimal") ||
// useTypeName(signature, "decimal") ||
useTypeName(signature, "timestamp with time zone") ||
useTypeName(signature, "interval day to second") ||
(!enableDecimalType && useTypeName(signature, "decimal")) ||
(!enableComplexType && useComplexType) ||
(enableComplexType && useTypeName(signature, "unknown")));
}

Expand Down Expand Up @@ -751,10 +755,10 @@ ExpressionFuzzer::ExpressionFuzzer(
for (const auto& signature : function.second) {
++totalFunctionSignatures;

if (!isSupportedSignature(*signature, options_.enableComplexTypes)) {
if (!isSupportedSignature(*signature, options_.enableComplexTypes, options_.enableDecimalType)) {
continue;
}
if (!(signature->variables().empty() || options_.enableComplexTypes)) {
if (!(signature->variables().empty() || options_.enableComplexTypes || options_.enableDecimalType)) {
LOG(WARNING) << "Skipping unsupported signature: " << function.first
<< signature->toString();
continue;
Expand Down Expand Up @@ -894,8 +898,10 @@ ExpressionFuzzer::ExpressionFuzzer(
sortSignatureTemplates(signatureTemplates_);

for (const auto& it : signatureTemplates_) {
auto& returnType = it.signature->returnType().baseName();
auto* returnTypeKey = &returnType;
const auto returnType = exec::sanitizeName(
it.signature->returnType().baseName());
const auto* returnTypeKey = &returnType;

if (it.typeVariables.find(returnType) != it.typeVariables.end()) {
// Return type is a template variable.
returnTypeKey = &kTypeParameterName;
Expand Down Expand Up @@ -969,7 +975,7 @@ void ExpressionFuzzer::addToTypeToExpressionListByTicketTimes(
const std::string& funcName) {
int tickets = getTickets(funcName);
for (int i = 0; i < tickets; i++) {
typeToExpressionList_[type].push_back(funcName);
typeToExpressionList_[exec::sanitizeName(type)].push_back(funcName);
}
}

Expand Down Expand Up @@ -1243,7 +1249,7 @@ core::TypedExprPtr ExpressionFuzzer::generateExpression(
} else {
expression = generateExpressionFromConcreteSignatures(
returnType, chosenFunctionName);
if (!expression && options_.enableComplexTypes) {
if (!expression && (options_.enableComplexTypes || options_.enableDecimalType)) {
expression = generateExpressionFromSignatureTemplate(
returnType, chosenFunctionName);
}
Expand Down
4 changes: 4 additions & 0 deletions velox/expression/tests/ExpressionFuzzer.h
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,10 @@ class ExpressionFuzzer {
// types.
bool enableComplexTypes = false;

// Enable testing of function signatures with decimal argument or return
// types.
bool enableDecimalType = false;

// Enable generation of expressions where one input column can be used by
// multiple subexpressions.
bool enableColumnReuse = false;
Expand Down
6 changes: 6 additions & 0 deletions velox/expression/tests/FuzzerRunner.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,11 @@ DEFINE_bool(
false,
"Enable testing of function signatures with complex argument or return types.");

DEFINE_bool(
velox_fuzzer_enable_decimal_type,
false,
"Enable testing of function signatures with decimal argument or return types.");

DEFINE_bool(
velox_fuzzer_enable_column_reuse,
false,
Expand Down Expand Up @@ -168,6 +173,7 @@ ExpressionFuzzer::Options getExpressionFuzzerOptions(
opts.enableVariadicSignatures = FLAGS_enable_variadic_signatures;
opts.enableDereference = FLAGS_enable_dereference;
opts.enableComplexTypes = FLAGS_velox_fuzzer_enable_complex_types;
opts.enableDecimalType = FLAGS_velox_fuzzer_enable_decimal_type;
opts.enableColumnReuse = FLAGS_velox_fuzzer_enable_column_reuse;
opts.enableExpressionReuse = FLAGS_velox_fuzzer_enable_expression_reuse;
opts.functionTickets = FLAGS_assign_function_tickets;
Expand Down
24 changes: 0 additions & 24 deletions velox/vector/fuzzer/VectorFuzzer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1040,32 +1040,8 @@ const std::vector<TypePtr> defaultScalarTypes() {
};
return kScalarTypes;
}

std::pair<int8_t, int8_t> randPrecisionScale(
FuzzerGenerator& rng,
int8_t maxPrecision) {
// Generate precision in range [1, Decimal type max precision]
auto precision = 1 + rand<int8_t>(rng) % maxPrecision;
// Generate scale in range [0, precision]
auto scale = rand<int8_t>(rng) % (precision + 1);
return {precision, scale};
}

} // namespace

TypePtr randDecimalType(FuzzerGenerator& rng) {
// Short or Long?
if (rand<bool>(rng)) {
auto [precision, scale] =
randPrecisionScale(rng, ShortDecimalType::kMaxPrecision);
return DECIMAL(precision, scale);
} else {
auto [precision, scale] =
randPrecisionScale(rng, LongDecimalType::kMaxPrecision);
return DECIMAL(precision, scale);
}
}

TypePtr randType(FuzzerGenerator& rng, int maxDepth) {
return randType(rng, defaultScalarTypes(), maxDepth);
}
Expand Down
2 changes: 0 additions & 2 deletions velox/vector/fuzzer/VectorFuzzer.h
Original file line number Diff line number Diff line change
Expand Up @@ -379,6 +379,4 @@ RowTypePtr randRowType(
const std::vector<TypePtr>& scalarTypes,
int maxDepth = 5);

TypePtr randDecimalType(FuzzerGenerator& rng);

} // namespace facebook::velox

0 comments on commit 0c1c343

Please sign in to comment.