Skip to content

Commit

Permalink
Add libfuzzer compatible fuzz harness
Browse files Browse the repository at this point in the history 
As discussed in #4088 the fuzz_simplify harness is an easy
candidate to begin support for coverage driven feedback
that is required for high performance fuzz testing.
  • Loading branch information
@nathaniel-brough
nathaniel-brough committed Apr 15, 2023
1 parent 56490e0 commit 1049fc5
Show file tree
Hide file tree
Showing 4 changed files with 85 additions and 67 deletions.
15 changes: 15 additions & 0 deletions test/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
include(HalideTestHelpers)
include(CheckCXXCompilerFlag)

# Internal tests are a special case.
# HalideTestHelpers depends on this test being present.
Expand Down Expand Up @@ -61,3 +62,17 @@ else ()
endif ()

# FIXME: failing_with_issue is dead code :)

check_cxx_compiler_flag("-fsanitize=fuzzer" HAS_FUZZ_FLAGS)

cmake_dependent_option(
WITH_TEST_FUZZ "Build fuzz tests" ON
HAS_FUZZ_FLAGS ON
)

if (WITH_TEST_FUZZ)
message(STATUS "Building fuzz tests enabled")
add_subdirectory(fuzz)
else ()
message(STATUS "Building fuzz tests disabled")
endif ()
1 change: 0 additions & 1 deletion test/correctness/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,6 @@ tests(GROUPS correctness
fuzz_bounds.cpp
fuzz_cse.cpp
fuzz_float_stores.cpp
fuzz_simplify.cpp
gameoflife.cpp
gather.cpp
gpu_allocation_cache.cpp
Expand Down
15 changes: 15 additions & 0 deletions test/fuzz/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
tests(GROUPS fuzz
SOURCES
simplify.cpp
)


# Allow OSS-fuzz to manage flags directly
if (DEFINED ENV{LIB_FUZZING_ENGINE})
target_link_options(fuzz_simplify PRIVATE ENV(LIB_FUZZING_ENGINE))
else ()
# By default just build with address sanitizers/libfuzzer for local testing
target_compile_options(fuzz_simplify PRIVATE -fsanitize=fuzzer,address)
target_link_options(fuzz_simplify PRIVATE -fsanitize=fuzzer,address)
endif ()

121 changes: 55 additions & 66 deletions test/correctness/fuzz_simplify.cpp → test/fuzz/simplify.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#include "Halide.h"
#include <array>
#include <fuzzer/FuzzedDataProvider.h>
#include <random>
#include <stdio.h>
#include <time.h>
Expand All @@ -14,76 +15,73 @@ using namespace Halide::Internal;

const int fuzz_var_count = 5;

// use std::mt19937 instead of rand() to ensure consistent behavior on all systems
std::mt19937 rng(0);

Type fuzz_types[] = {UInt(1), UInt(8), UInt(16), UInt(32), Int(8), Int(16), Int(32)};
const int fuzz_type_count = sizeof(fuzz_types) / sizeof(fuzz_types[0]);

std::string fuzz_var(int i) {
return std::string(1, 'a' + i);
}

Expr random_var() {
int fuzz_count = rng() % fuzz_var_count;
Expr random_var(FuzzedDataProvider &fdp) {
int fuzz_count = fdp.ConsumeIntegralInRange<int>(0, fuzz_var_count);
return Variable::make(Int(0), fuzz_var(fuzz_count));
}

Type random_type(int width) {
Type T = fuzz_types[rng() % fuzz_type_count];
Type random_type(FuzzedDataProvider &fdp, int width) {
Type T = fdp.PickValueInArray(fuzz_types);

if (width > 1) {
T = T.with_lanes(width);
}
return T;
}

int get_random_divisor(Type t) {
int get_random_divisor(FuzzedDataProvider &fdp, Type t) {
std::vector<int> divisors = {t.lanes()};
for (int dd = 2; dd < t.lanes(); dd++) {
if (t.lanes() % dd == 0) {
divisors.push_back(dd);
}
}

return divisors[rng() % divisors.size()];
return divisors[fdp.ConsumeIntegralInRange<size_t>(0, divisors.size()-1)];
}

Expr random_leaf(Type T, bool overflow_undef = false, bool imm_only = false) {
Expr random_leaf(FuzzedDataProvider &fdp, Type T, bool overflow_undef = false, bool imm_only = false) {
if (T.is_int() && T.bits() == 32) {
overflow_undef = true;
}
if (T.is_scalar()) {
int var = rng() % fuzz_var_count + 1;
int var = fdp.ConsumeIntegralInRange<int>(0, fuzz_var_count) + 1;
if (!imm_only && var < fuzz_var_count) {
auto v1 = random_var();
auto v1 = random_var(fdp);
return cast(T, v1);
} else {
if (overflow_undef) {
// For Int(32), we don't care about correctness during
// overflow, so just use numbers that are unlikely to
// overflow.
return cast(T, (int)(rng() % 256 - 128));
return cast(T, fdp.ConsumeIntegralInRange<int>(-128, 128));
} else {
return cast(T, (int)(rng() - RAND_MAX / 2));
return cast(T, fdp.ConsumeIntegralInRange<int>(-RAND_MAX / 2, RAND_MAX / 2));
}
}
} else {
int lanes = get_random_divisor(T);
if (rng() % 2 == 0) {
auto e1 = random_leaf(T.with_lanes(T.lanes() / lanes), overflow_undef);
auto e2 = random_leaf(T.with_lanes(T.lanes() / lanes), overflow_undef);
int lanes = get_random_divisor(fdp, T);
if (fdp.ConsumeBool()) {
auto e1 = random_leaf(fdp, T.with_lanes(T.lanes() / lanes), overflow_undef);
auto e2 = random_leaf(fdp, T.with_lanes(T.lanes() / lanes), overflow_undef);
return Ramp::make(e1, e2, lanes);
} else {
auto e1 = random_leaf(T.with_lanes(T.lanes() / lanes), overflow_undef);
auto e1 = random_leaf(fdp, T.with_lanes(T.lanes() / lanes), overflow_undef);
return Broadcast::make(e1, lanes);
}
}
}

Expr random_expr(Type T, int depth, bool overflow_undef = false);
Expr random_expr(FuzzedDataProvider &fdp, Type T, int depth, bool overflow_undef = false);

Expr random_condition(Type T, int depth, bool maybe_scalar) {
Expr random_condition(FuzzedDataProvider &fdp, Type T, int depth, bool maybe_scalar) {
typedef Expr (*make_bin_op_fn)(Expr, Expr);
static make_bin_op_fn make_bin_op[] = {
EQ::make,
Expand All @@ -95,13 +93,13 @@ Expr random_condition(Type T, int depth, bool maybe_scalar) {
};
const int op_count = sizeof(make_bin_op) / sizeof(make_bin_op[0]);

if (maybe_scalar && rng() % T.lanes() == 0) {
if (maybe_scalar && fdp.ConsumeBool()) {
T = T.element_of();
}

Expr a = random_expr(T, depth);
Expr b = random_expr(T, depth);
int op = rng() % op_count;
Expr a = random_expr(fdp, T, depth);
Expr b = random_expr(fdp, T, depth);
int op = fdp.ConsumeIntegralInRange<int>(0, op_count);
return make_bin_op[op](a, b);
}

Expand All @@ -111,7 +109,7 @@ Expr make_absd(Expr a, Expr b) {
return cast(a.type(), absd(a, b));
}

Expr random_expr(Type T, int depth, bool overflow_undef) {
Expr random_expr(FuzzedDataProvider &fdp, Type T, int depth, bool overflow_undef) {
typedef Expr (*make_bin_op_fn)(Expr, Expr);
static make_bin_op_fn make_bin_op[] = {
Add::make,
Expand All @@ -134,60 +132,60 @@ Expr random_expr(Type T, int depth, bool overflow_undef) {
}

if (depth-- <= 0) {
return random_leaf(T, overflow_undef);
return random_leaf(fdp, T, overflow_undef);
}

const int bin_op_count = sizeof(make_bin_op) / sizeof(make_bin_op[0]);
const int bool_bin_op_count = sizeof(make_bool_bin_op) / sizeof(make_bool_bin_op[0]);
const int op_count = bin_op_count + bool_bin_op_count + 5;

int op = rng() % op_count;
int op = fdp.ConsumeIntegralInRange<int>(0, op_count);
switch (op) {
case 0:
return random_leaf(T);
return random_leaf(fdp, T);
case 1: {
auto c = random_condition(T, depth, true);
auto e1 = random_expr(T, depth, overflow_undef);
auto e2 = random_expr(T, depth, overflow_undef);
auto c = random_condition(fdp, T, depth, true);
auto e1 = random_expr(fdp, T, depth, overflow_undef);
auto e2 = random_expr(fdp, T, depth, overflow_undef);
return Select::make(c, e1, e2);
}
case 2:
if (T.lanes() != 1) {
int lanes = get_random_divisor(T);
auto e1 = random_expr(T.with_lanes(T.lanes() / lanes), depth, overflow_undef);
int lanes = get_random_divisor(fdp, T);
auto e1 = random_expr(fdp, T.with_lanes(T.lanes() / lanes), depth, overflow_undef);
return Broadcast::make(e1, lanes);
}
break;
case 3:
if (T.lanes() != 1) {
int lanes = get_random_divisor(T);
auto e1 = random_expr(T.with_lanes(T.lanes() / lanes), depth, overflow_undef);
auto e2 = random_expr(T.with_lanes(T.lanes() / lanes), depth, overflow_undef);
int lanes = get_random_divisor(fdp, T);
auto e1 = random_expr(fdp, T.with_lanes(T.lanes() / lanes), depth, overflow_undef);
auto e2 = random_expr(fdp, T.with_lanes(T.lanes() / lanes), depth, overflow_undef);
return Ramp::make(e1, e2, lanes);
}
break;

case 4:
if (T.is_bool()) {
auto e1 = random_expr(T, depth);
auto e1 = random_expr(fdp, T, depth);
return Not::make(e1);
}
break;

case 5:
// When generating boolean expressions, maybe throw in a condition on non-bool types.
if (T.is_bool()) {
return random_condition(random_type(T.lanes()), depth, false);
return random_condition(fdp, random_type(fdp, T.lanes()), depth, false);
}
break;

case 6: {
// Get a random type that isn't T or int32 (int32 can overflow and we don't care about that).
Type subT;
do {
subT = random_type(T.lanes());
subT = random_type(fdp, T.lanes());
} while (subT == T || (subT.is_int() && subT.bits() == 32));
auto e1 = random_expr(subT, depth, overflow_undef);
auto e1 = random_expr(fdp, subT, depth, overflow_undef);
return Cast::make(T, e1);
}

Expand All @@ -198,12 +196,12 @@ Expr random_expr(Type T, int depth, bool overflow_undef) {
} else {
maker = make_bin_op[op % bin_op_count];
}
Expr a = random_expr(T, depth, overflow_undef);
Expr b = random_expr(T, depth, overflow_undef);
Expr a = random_expr(fdp, T, depth, overflow_undef);
Expr b = random_expr(fdp, T, depth, overflow_undef);
return maker(a, b);
}
// If we got here, try again.
return random_expr(T, depth, overflow_undef);
return random_expr(fdp, T, depth, overflow_undef);
}

bool test_simplification(Expr a, Expr b, Type T, const map<string, Expr> &vars) {
Expand Down Expand Up @@ -238,7 +236,7 @@ bool test_simplification(Expr a, Expr b, Type T, const map<string, Expr> &vars)
return true;
}

bool test_expression(Expr test, int samples) {
bool test_expression(FuzzedDataProvider &fdp, Expr test, int samples) {
Expr simplified = simplify(test);

map<string, Expr> vars;
Expand All @@ -248,10 +246,13 @@ bool test_expression(Expr test, int samples) {

for (int i = 0; i < samples; i++) {
for (std::map<string, Expr>::iterator v = vars.begin(); v != vars.end(); v++) {
size_t kMaxLeafIterations = 10000;
// Don't let the random leaf depend on v itself.
size_t iterations = 0;
do {
v->second = random_leaf(test.type().element_of(), true);
} while (expr_uses_var(v->second, v->first));
v->second = random_leaf(fdp, test.type().element_of(), true);
iterations++;
} while (expr_uses_var(v->second, v->first) && iterations < kMaxLeafIterations);
}

if (!test_simplification(test, simplified, test.type(), vars)) {
Expand Down Expand Up @@ -334,31 +335,19 @@ Expr e(Variable::make(Int(0), fuzz_var(4)));

} // namespace

int main(int argc, char **argv) {
// Number of random expressions to test.
const int count = 10000;
extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
// Depth of the randomly generated expression trees.
const int depth = 5;
// Number of samples to test the generated expressions for.
const int samples = 3;

// We want different fuzz tests every time, to increase coverage.
// We also report the seed to enable reproducing failures.
int fuzz_seed = argc > 1 ? atoi(argv[1]) : time(nullptr);
rng.seed(fuzz_seed);
std::cout << "Simplify fuzz test seed: " << fuzz_seed << "\n";
FuzzedDataProvider fdp(data, size);

std::array<int, 6> vector_widths = {1, 2, 3, 4, 6, 8};
for (int n = 0; n < count; n++) {
int width = vector_widths[rng() % vector_widths.size()];
Type VT = random_type(width);
// Generate a random expr...
Expr test = random_expr(VT, depth);
if (!test_expression(test, samples)) {
return 1;
}
}

std::cout << "Success!\n";
int width = fdp.PickValueInArray(vector_widths);
Type VT = random_type(fdp, width);
// Generate a random expr...
Expr test = random_expr(fdp, VT, depth);
assert(test_expression(fdp, test, samples));
return 0;
}

0 comments on commit 1049fc5

Please sign in to comment.