build: Fix cifuzz tests and improve fuzz tests' reliability

controllers/kustomization_fuzzer_test.go

@@ -1,5 +1,5 @@
-//go:build gofuzz
-// +build gofuzz
+//go:build gofuzz_libfuzzer
+// +build gofuzz_libfuzzer
 
 /*
 Copyright 2021 The Flux authors
@@ -20,28 +20,47 @@ limitations under the License.
 package controllers
 
 import (
+	"archive/tar"
+	"compress/gzip"
 	"context"
+	"crypto/sha1"
+	"crypto/sha256"
+	"embed"
 	"errors"
 	"fmt"
+	"io"
+	"io/fs"
+	"math/rand"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"sync"
+	"testing"
 	"time"
 
-	"embed"
-	"io/fs"
-
 	securejoin "github.com/cyphar/filepath-securejoin"
-	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta2"
-	"github.com/fluxcd/pkg/apis/meta"
-	"github.com/fluxcd/pkg/runtime/testenv"
-	sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
+	"github.com/hashicorp/vault/api"
+	"github.com/ory/dockertest"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/client-go/kubernetes/scheme"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
+	controllerLog "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+
+	"github.com/fluxcd/pkg/apis/meta"
+	"github.com/fluxcd/pkg/runtime/controller"
+	"github.com/fluxcd/pkg/runtime/testenv"
+	"github.com/fluxcd/pkg/testserver"
+	sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
 
 	fuzz "github.com/AdaLogics/go-fuzz-headers"
+	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta2"
 )
 
 var (

tests/fuzz/README.md

@@ -39,7 +39,23 @@ Run fuzzer inside a container:
 		/out/fuzz_conditions_match
 ```
 
+### Running oss-fuzz locally
+
+The `make fuzz-smoketest` is meant to be an easy way to reproduce errors that may occur
+upstream. If our checks ever run out of sync with upstream, the upstream tests can be
+executed locally with:
+
+```
+git clone --depth 1 https://github.com/google/oss-fuzz
+cd oss-fuzz
+python infra/helper.py build_image fluxcd
+python infra/helper.py build_fuzzers --sanitizer address --architecture x86_64 fluxcd
+python infra/helper.py check_build --sanitizer address --architecture x86_64 fluxcd
+```
+
+For latest info on testing oss-fuzz locally, refer to the [upstream guide].
 
 [oss fuzz]: https://github.com/google/oss-fuzz
 [oss-fuzz repository]: https://github.com/google/oss-fuzz/tree/master/projects/fluxcd
 [github workflow]: .github/workflows/cifuzz.yaml
+[upstream guide]: https://google.github.io/oss-fuzz/getting-started/new-project-guide/#testing-locally

tests/fuzz/oss_fuzz_build.sh

@@ -16,69 +16,66 @@
 
 set -euxo pipefail
 
+# This file aims for:
+# - Dynamically discover and build all fuzz tests within the repository.
+# - Work for both local make fuzz-smoketest and the upstream oss-fuzz.
+
 GOPATH="${GOPATH:-/root/go}"
 GO_SRC="${GOPATH}/src"
 PROJECT_PATH="github.com/fluxcd/kustomize-controller"
-TMP_DIR=$(mktemp -d /tmp/oss_fuzz-XXXXXX)
-
-cleanup(){
-	rm -rf "${TMP_DIR}"
-}
-trap cleanup EXIT
 
+# install_deps installs all dependencies needed for upstream oss-fuzz.
+# Unfortunately we can't pin versions here, as we want to always
+# have the latest, so that we can reproduce errors occuring upstream.
 install_deps(){
-	if ! command -v go-118-fuzz-build &> /dev/null || ! command -v addimport &> /dev/null; then
-		mkdir -p "${TMP_DIR}/go-118-fuzz-build"
-
-		git clone https://github.com/AdamKorcz/go-118-fuzz-build "${TMP_DIR}/go-118-fuzz-build"
-		cd "${TMP_DIR}/go-118-fuzz-build"
-		go build -o "${GOPATH}/bin/go-118-fuzz-build"
-
-		cd addimport
-		go build -o "${GOPATH}/bin/addimport"
-	fi
-
-	if ! command -v goimports &> /dev/null; then
-		go install golang.org/x/tools/cmd/goimports@latest
+	if ! command -v go-118-fuzz-build &> /dev/null; then
+		go install github.com/AdamKorcz/go-118-fuzz-build@latest
 	fi
 }
 
-# Removes the content of test funcs which could cause the Fuzz
-# tests to break.
-remove_test_funcs(){
-	filename=$1
-
-	echo "removing co-located *testing.T"
-	sed -i -e '/func Test.*testing.T) {$/ {:r;/\n}/!{N;br}; s/\n.*\n/\n/}' "${filename}"
-	# Remove gomega reference as it is not used by Fuzz tests.
-	sed -i 's;. "github.com/onsi/gomega";;g' "${filename}"
-
-	# After removing the body of the go testing funcs, consolidate the imports.
-	goimports -w "${filename}"
-}
-
 install_deps
 
 cd "${GO_SRC}/${PROJECT_PATH}"
 
-go get github.com/AdamKorcz/go-118-fuzz-build/utils
+# Ensure any project-specific requirements are catered for ahead of
+# the generic build process.
+if [ -f "tests/fuzz/oss_fuzz_prebuild.sh" ]; then
+	tests/fuzz/oss_fuzz_prebuild.sh
+fi
+
+modules=$(find . -mindepth 1 -maxdepth 4 -type f -name 'go.mod' | cut -c 3- | sed 's|/[^/]*$$||' | sort -u | sed 's;/go.mod;;g' | sed 's;go.mod;.;g')
 
-mkdir -p controllers/testdata/crd
-cp config/crd/bases/*.yaml controllers/testdata/crd
+for module in ${modules}; do
 
-# Iterate through all Go Fuzz targets, compiling each into a fuzzer.
-test_files=$(grep -r --include='**_test.go' --files-with-matches 'func Fuzz' .)
-for file in ${test_files}
-do
-	remove_test_funcs "${file}"
+	cd "${GO_SRC}/${PROJECT_PATH}/${module}"
 
-	targets=$(grep -oP 'func \K(Fuzz\w*)' "${file}")
-	for target_name in ${targets}
-	do
-		fuzzer_name=$(echo "${target_name}" | tr '[:upper:]' '[:lower:]')
-		target_dir=$(dirname "${file}")
+	# TODO: stop ignoring recorder_fuzzer_test.go. Temporary fix for fuzzing building issues.
+	test_files=$(grep -r --include='**_test.go' --files-with-matches 'func Fuzz' . || echo "")
+	if [ -z "${test_files}" ]; then
+		continue
+	fi
 
-		echo "Building ${file}.${target_name} into ${fuzzer_name}"
-		compile_native_go_fuzzer "${target_dir}" "${target_name}" "${fuzzer_name}"
+	go get github.com/AdamKorcz/go-118-fuzz-build/testing
+
+	# Iterate through all Go Fuzz targets, compiling each into a fuzzer.
+	for file in ${test_files}; do
+		# If the subdir is a module, skip this file, as it will be handled
+		# at the next iteration of the outer loop. 
+		if [ -f "$(dirname "${file}")/go.mod" ]; then
+			continue
+		fi
+
+		targets=$(grep -oP 'func \K(Fuzz\w*)' "${file}")
+		for target_name in ${targets}; do
+			# Transform module path into module name (e.g. git/libgit2 to git_libgit2).
+			module_name=$(echo ${module} | tr / _)
+			# Compose fuzzer name based on the lowercase version of the func names.
+			# The module name is added after the fuzz prefix, for better discoverability.
+			fuzzer_name=$(echo "${target_name}" | tr '[:upper:]' '[:lower:]' | sed "s;fuzz_;fuzz_${module_name}_;g")
+			target_dir=$(dirname "${file}")
+
+			echo "Building ${file}.${target_name} into ${fuzzer_name}"
+			compile_native_go_fuzzer "${target_dir}" "${target_name}" "${fuzzer_name}"
+		done
 	done
 done

tests/fuzz/oss_fuzz_prebuild.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# Copyright 2022 The Flux authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euxo pipefail
+
+# This file is executed by upstream oss-fuzz for any requirements that
+# are specific for building this project.
+
+# Some tests requires embedded resources. Embedding does not allow
+# for traversing into ascending dirs, therefore we copy those contents here:
+mkdir -p controllers/testdata/crd
+cp config/crd/bases/*.yaml controllers/testdata/crd