From 88a5716f4241192ec4d045f73c78faaf0a522f48 Mon Sep 17 00:00:00 2001
From: Jou Ho <jouh@amazon.com>
Date: Fri, 6 Dec 2024 20:17:46 +0000
Subject: [PATCH 01/10] make fuzz tests run in parallel and generate coverage
 report

---
 CMakeLists.txt                              | 10 ++-
 codebuild/spec/buildspec_fuzz.yml           |  2 +-
 codebuild/spec/buildspec_fuzz_scheduled.yml | 73 +++++++--------------
 tests/fuzz/calcTotalCov.sh                  | 38 ++++-------
 tests/fuzz/runFuzzTest.sh                   | 22 ++++---
 5 files changed, 56 insertions(+), 89 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f19e25d1440..b1144ea69d3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -246,7 +246,7 @@ if (COVERAGE)
     # on LLVM compilers. GCC would fail with "unrecognized compile options"
     # on -fprofile-instr-generate -fcoverage-mapping flags.
     if (NOT ${CMAKE_C_COMPILER_ID} MATCHES Clang)
-        message(FATAL_ERROR "This project requires clang for coverage support")
+        message(FATAL_ERROR "This project requires clang for coverage support. You are currently using " ${CMAKE_C_COMPILER_ID})
     endif()
     target_compile_options(${PROJECT_NAME} PUBLIC -fprofile-instr-generate -fcoverage-mapping)
     target_link_options(${PROJECT_NAME} PUBLIC -fprofile-instr-generate -fcoverage-mapping)
@@ -696,6 +696,12 @@ if (BUILD_TESTING)
             set(ARTIFACT_UPLOAD_LOC "none")
         endif()
 
+        if(COVERAGE)
+            set(FUZZ_COVERAGE "true")
+        else()
+            set(FUZZ_COVERAGE "false")
+        endif()
+
         # Build LD_PRELOAD shared libraries
         file(GLOB LIBRARY_SRCS "${CMAKE_CURRENT_SOURCE_DIR}/tests/fuzz/LD_PRELOAD/*.c")
         foreach(SRC ${LIBRARY_SRCS})
@@ -732,6 +738,8 @@ if (BUILD_TESTING)
                     ${BUILD_DIR_PATH} 
                     ${CORPUS_UPLOAD_LOC} 
                     ${ARTIFACT_UPLOAD_LOC}
+                    ${FUZZ_COVERAGE}
+                    ${CMAKE_CURRENT_SOURCE_DIR}
                 WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests/fuzz
             )
             set_property(TEST ${TEST_NAME} PROPERTY LABELS "fuzz")
diff --git a/codebuild/spec/buildspec_fuzz.yml b/codebuild/spec/buildspec_fuzz.yml
index c74700ff28f..182238809a4 100644
--- a/codebuild/spec/buildspec_fuzz.yml
+++ b/codebuild/spec/buildspec_fuzz.yml
@@ -33,4 +33,4 @@ phases:
     on-failure: ABORT
     commands:
       # -L: Restrict tests to names matching the pattern 'fuzz'
-      - cmake --build build/ --target test -- ARGS="-L fuzz --output-on-failure"
+      - cmake --build build/ --target test -- ARGS="-L fuzz --output-on-failure -j (nproc)"
diff --git a/codebuild/spec/buildspec_fuzz_scheduled.yml b/codebuild/spec/buildspec_fuzz_scheduled.yml
index b7672a4e2c9..5f26166a943 100644
--- a/codebuild/spec/buildspec_fuzz_scheduled.yml
+++ b/codebuild/spec/buildspec_fuzz_scheduled.yml
@@ -13,55 +13,16 @@
 # limitations under the License.
 version: 0.2
 
-batch:
-  build-matrix:
-    static:
-      env:
-        privileged-mode: true
-    dynamic:
-      env:
-        compute-type:
-          - BUILD_GENERAL1_LARGE
-        image:
-          - 024603541914.dkr.ecr.us-west-2.amazonaws.com/docker:ubuntu22codebuild
-        privileged-mode: true
-        variables:
-          S2N_LIBCRYPTO: 
-            - awslc
-          FUZZ_TESTS:
-            - "s2n_cert_req_recv_test"
-            - "s2n_certificate_extensions_parse_test"
-            - "s2n_client_ccs_recv_test"
-            - "s2n_client_cert_recv_test"
-            - "s2n_client_cert_verify_recv_test"
-            - "s2n_client_finished_recv_test"
-            - "s2n_client_fuzz_test"
-            - "s2n_client_hello_recv_fuzz_test"
-            - "s2n_client_key_recv_fuzz_test"
-            - "s2n_deserialize_resumption_state_test"
-            - "s2n_encrypted_extensions_recv_test"
-            - "s2n_extensions_client_key_share_recv_test"
-            - "s2n_extensions_client_supported_versions_recv_test"
-            - "s2n_extensions_server_key_share_recv_test"
-            - "s2n_extensions_server_supported_versions_recv_test"
-            - "s2n_hybrid_ecdhe_kyber_r3_fuzz_test"
-            - "s2n_kyber_r3_recv_ciphertext_fuzz_test"
-            - "s2n_kyber_r3_recv_public_key_fuzz_test"
-            - "s2n_memory_leak_negative_test"
-            - "s2n_openssl_diff_pem_parsing_test"
-            - "s2n_recv_client_supported_groups_test"
-            - "s2n_select_server_cert_test"
-            - "s2n_server_ccs_recv_test"
-            - "s2n_server_cert_recv_test"
-            - "s2n_server_extensions_recv_test"
-            - "s2n_server_finished_recv_test"
-            - "s2n_server_fuzz_test"
-            - "s2n_server_hello_recv_test"
-            - "s2n_stuffer_pem_fuzz_test"
-            - "s2n_tls13_cert_req_recv_test"
-            - "s2n_tls13_cert_verify_recv_test"
-            - "s2n_tls13_client_finished_recv_test"
-            - "s2n_tls13_server_finished_recv_test"
+env:
+  compute-type:
+    - BUILD_GENERAL1_LARGE
+  image:
+    - 024603541914.dkr.ecr.us-west-2.amazonaws.com/docker:ubuntu22codebuild
+  privileged-mode: true
+  variables:
+    CC: "/usr/bin/clang"
+    CXX: "/usr/bin/clang++"
+    S2N_LIBCRYPTO: "awslc"
 
 phases:
   pre_build:
@@ -73,11 +34,14 @@ phases:
   build:
     on-failure: ABORT
     commands:
+      #TODO: change fuzz timeout back to 27000
       - |
         cmake . -Bbuild \
         -DCMAKE_PREFIX_PATH=/usr/local/$S2N_LIBCRYPTO \
         -DS2N_FUZZ_TEST=on \
-        -DFUZZ_TIMEOUT_SEC=27000
+        -DFUZZ_TIMEOUT_SEC=60 \
+        -DCOVERAGE=on \
+        -DBUILD_SHARED_LIBS=ON
       - cmake --build ./build -- -j $(nproc)
   post_build:
     on-failure: ABORT
@@ -85,4 +49,11 @@ phases:
       # -L: Restrict tests to labels matching the pattern 'fuzz'
       # -R: Run the single fuzz test defined in ${FUZZ_TESTS}
       # --timeout: override ctest's default timeout of 1500
-      - cmake --build build/ --target test -- ARGS="-L fuzz -R ${FUZZ_TESTS} --output-on-failure --timeout 28800"
+      - cmake --build build/ --target test -- ARGS="-L fuzz -R ${FUZZ_TESTS} --output-on-failure -j $(nproc) --timeout 28800"
+      - ./tests/fuzz/calcTotalCov.sh
+
+artifacts:
+# upload all files in the coverage_report directory
+files:
+  - '**/*'
+base-directory: fuzz_coverage_report
diff --git a/tests/fuzz/calcTotalCov.sh b/tests/fuzz/calcTotalCov.sh
index f36255e9b47..66279cd4e1c 100755
--- a/tests/fuzz/calcTotalCov.sh
+++ b/tests/fuzz/calcTotalCov.sh
@@ -24,36 +24,20 @@ if [ "$#" -ne "0" ]; then
     usage
 fi
 
-if [[ -z "$S2N_ROOT" ]]; then
-    S2N_ROOT=../..
-fi
-
-FUZZCOV_SOURCES="${S2N_ROOT}/api ${S2N_ROOT}/bin ${S2N_ROOT}/crypto ${S2N_ROOT}/error ${S2N_ROOT}/stuffer ${S2N_ROOT}/tls ${S2N_ROOT}/utils"
+FUZZCOV_SOURCES="api bin crypto error stuffer tls utils"
 
+printf "Calculating total s2n coverage... "
 
-# Outputs fuzz coverage results if the FUZZ_COVERAGE environment variable is set
-# Total coverage is overlayed on source code in s2n_cov.html and coverage statistics are available in s2n_cov.txt
-# If using LLVM version 9 or greater, coverage is output in LCOV format instead of HTML
-# All files are stored in the s2n coverage directory
-if [[ "$FUZZ_COVERAGE" == "true" ]]; then
+# The llvm-profdata merge command warns that the profraws were created from different binaries (which is true) but
+# works fine for what we care about (the s2n library). Therefore, for user clarity all output is suppressed.
+llvm-profdata merge -sparse tests/fuzz/profiles/*/*.profdata -o tests/fuzz/profiles/merged_fuzz.profdata > /dev/null 2>&1
 
-    printf "Calculating total s2n coverage... "
+llvm-cov report -instr-profile=tests/fuzz/profiles/merged_fuzz.profdata build/lib/libs2n.so ${FUZZCOV_SOURCES} > s2n_fuzz_coverage.txt
 
-    # The llvm-profdata merge command warns that the profraws were created from different binaries (which is true) but
-    # works fine for what we care about (the s2n library). Therefore, for user clarity all output is suppressed.
-    llvm-profdata merge -sparse ./profiles/*/*.profdata -o ./profiles/s2n_cov.profdata > /dev/null 2>&1
-    llvm-cov report -instr-profile=./profiles/s2n_cov.profdata ${S2N_ROOT}/lib/libs2n.so ${FUZZCOV_SOURCES} > ${COVERAGE_DIR}/fuzz/s2n_cov.txt
+# convert coverage information to html format
+llvm-cov export -instr-profile=tests/fuzz/profiles/merged_fuzz.profdata build/lib/libs2n.so ${FUZZCOV_SOURCES} -format=lcov > s2n_fuzz_cov.info
 
-    # Use LCOV format instead of HTML if the LLVM version we're using supports it
-    if [[ $(grep -Eo "[0-9]*" <<< `llvm-cov --version` | head -1) > 8 ]]; then
-        llvm-cov export -instr-profile=./profiles/s2n_cov.profdata ${S2N_ROOT}/lib/libs2n.so ${FUZZCOV_SOURCES} -format=lcov > ${COVERAGE_DIR}/fuzz/s2n_cov.info
-        genhtml -q -o ${COVERAGE_DIR}/html/overall_fuzz_coverage ${COVERAGE_DIR}/fuzz/s2n_cov.info
-    else
-        llvm-cov show -instr-profile=./profiles/s2n_cov.profdata ${S2N_ROOT}/lib/libs2n.so ${FUZZCOV_SOURCES} -use-color -format=html > ${COVERAGE_DIR}/fuzz/s2n_cov.html
-    fi
-    # Generate coverage report compatible with codecov.io
-    llvm-cov show -instr-profile=./profiles/s2n_cov.profdata ${S2N_ROOT}/lib/libs2n.so ${FUZZCOV_SOURCES} > ${COVERAGE_DIR}/fuzz/codecov.txt
+genhtml s2n_fuzz_cov.info --branch-coverage -q -o fuzz_coverage_report
 
-    S2N_COV=`grep -Eo '[0-9]*\.[0-9]*\%' ${COVERAGE_DIR}/fuzz/s2n_cov.txt | tail -1`
-    printf "total s2n coverage from fuzz tests: %s\n" $S2N_COV
-fi
+S2N_COV=`grep -Eo '[0-9]*\.[0-9]*\%' s2n_fuzz_coverage.txt | tail -1`
+printf "total s2n coverage from fuzz tests: %s\n" $S2N_COV
diff --git a/tests/fuzz/runFuzzTest.sh b/tests/fuzz/runFuzzTest.sh
index 800135bca7b..d2d70b71a0f 100755
--- a/tests/fuzz/runFuzzTest.sh
+++ b/tests/fuzz/runFuzzTest.sh
@@ -18,11 +18,11 @@
 set -e
 
 usage() {
-    echo "Usage: runFuzzTest.sh TEST_NAME FUZZ_TIMEOUT_SEC"
+    echo "Usage: runFuzzTest.sh TEST_NAME FUZZ_TIMEOUT_SEC BUILD_DIR_PATH CORPUS_UPLOAD_LOC ARTIFACT_UPLOAD_LOC FUZZ_COVERAGE"
     exit 1
 }
 
-if [ "$#" -ne "5" ]; then
+if [ "$#" -ne "7" ]; then
     usage
 fi
 
@@ -31,6 +31,9 @@ FUZZ_TIMEOUT_SEC=$2
 BUILD_DIR_PATH=$3
 CORPUS_UPLOAD_LOC=$4
 ARTIFACT_UPLOAD_LOC=$5
+FUZZ_COVERAGE=$6
+S2N_ROOT=$7
+
 MIN_TEST_PER_SEC="1000"
 MIN_FEATURES_COVERED="100"
 
@@ -106,6 +109,7 @@ if [[ "$FUZZ_COVERAGE" == "true" ]]; then
     mkdir -p "./profiles/${TEST_NAME}"
     rm -f ./profiles/${TEST_NAME}/*.profraw
     LLVM_PROFILE_FILE="./profiles/${TEST_NAME}/${TEST_NAME}.%p.profraw" \
+    env LD_PRELOAD="$LD_PRELOAD_" \
     ${BUILD_DIR_PATH}/bin/${TEST_NAME} ${LIBFUZZER_ARGS} ${TEMP_CORPUS_DIR} \
     > ${TEST_NAME}_output.txt 2>&1 || ACTUAL_TEST_FAILURE=1
 else
@@ -129,16 +133,16 @@ declare -i TARGET_COV=0
 # If using LLVM version 9 or greater, coverage is output in LCOV format instead of HTML
 # All files are stored in the s2n coverage directory
 if [[ "$FUZZ_COVERAGE" == "true" ]]; then
-    mkdir -p ${COVERAGE_DIR}/fuzz
+    mkdir -p coverage/fuzz
     llvm-profdata merge -sparse ./profiles/${TEST_NAME}/*.profraw -o ./profiles/${TEST_NAME}/${TEST_NAME}.profdata
-    llvm-cov report -instr-profile=./profiles/${TEST_NAME}/${TEST_NAME}.profdata ${S2N_ROOT}/lib/libs2n.so ${FUZZCOV_SOURCES} -show-functions > ${COVERAGE_DIR}/fuzz/${TEST_NAME}_cov.txt
+    llvm-cov report -instr-profile=./profiles/${TEST_NAME}/${TEST_NAME}.profdata ${BUILD_DIR_PATH}/lib/libs2n.so ${FUZZCOV_SOURCES} -show-functions > coverage/fuzz/${TEST_NAME}_cov.txt
 
     # Use LCOV format instead of HTML if the LLVM version we're using supports it
     if [[ $(grep -Eo "[0-9]*" <<< `llvm-cov --version` | head -1) -gt 8 ]]; then
-        llvm-cov export -instr-profile=./profiles/${TEST_NAME}/${TEST_NAME}.profdata ${S2N_ROOT}/lib/libs2n.so ${FUZZCOV_SOURCES} -format=lcov > ${COVERAGE_DIR}/fuzz/${TEST_NAME}_cov.info
-        genhtml -q -o ${COVERAGE_DIR}/html/${TEST_NAME} ${COVERAGE_DIR}/fuzz/${TEST_NAME}_cov.info
+        llvm-cov export -instr-profile=./profiles/${TEST_NAME}/${TEST_NAME}.profdata ${BUILD_DIR_PATH}/lib/libs2n.so ${FUZZCOV_SOURCES} -format=lcov > coverage/fuzz/${TEST_NAME}_cov.info
+        genhtml -q -o coverage/html/${TEST_NAME} coverage/fuzz/${TEST_NAME}_cov.info
     else
-        llvm-cov show -instr-profile=./profiles/${TEST_NAME}/${TEST_NAME}.profdata ${S2N_ROOT}/lib/libs2n.so ${FUZZCOV_SOURCES} -use-color -format=html > ${COVERAGE_DIR}/fuzz/${TEST_NAME}_cov.html
+        llvm-cov show -instr-profile=./profiles/${TEST_NAME}/${TEST_NAME}.profdata ${BUILD_DIR_PATH}/lib/libs2n.so ${FUZZCOV_SOURCES} -use-color -format=html > coverage/fuzz/${TEST_NAME}_cov.html
     fi
 
     # Extract target functions from test source
@@ -149,8 +153,8 @@ if [[ "$FUZZ_COVERAGE" == "true" ]]; then
     then
         for TARGET in ${TARGET_FUNCS}
         do
-            TARGET_TOTAL+=`sed -n "s/^.*${TARGET} .*% *\([0-9]*\) .*$/\1/p" ${COVERAGE_DIR}/fuzz/${TEST_NAME}_cov.txt`
-            TARGET_COV+=`sed -n "s/^.*${TARGET} .*% *[0-9]* *\([0-9]*\) .*$/\1/p" ${COVERAGE_DIR}/fuzz/${TEST_NAME}_cov.txt`
+            TARGET_TOTAL+=`sed -n "s/^.*${TARGET} .*% *\([0-9]*\) .*$/\1/p" coverage/fuzz/${TEST_NAME}_cov.txt`
+            TARGET_COV+=`sed -n "s/^.*${TARGET} .*% *[0-9]* *\([0-9]*\) .*$/\1/p" coverage/fuzz/${TEST_NAME}_cov.txt`
         done
     fi
 fi

From 224a017760d66ba1b143616d6d3ac51094e8827e Mon Sep 17 00:00:00 2001
From: Jou Ho <jouh@amazon.com>
Date: Fri, 6 Dec 2024 20:26:42 +0000
Subject: [PATCH 02/10] fix buildspec syntax

run all fuzz tests

fix syntax error

run tests in parallel

modify coverage summary

modify coverage output

update comment

simplify output

test shorter runtime

remove comments
---
 codebuild/spec/buildspec_fuzz.yml           |  2 +-
 codebuild/spec/buildspec_fuzz_scheduled.yml | 19 ++++++-------------
 tests/fuzz/calcTotalCov.sh                  |  9 ++++++---
 tests/fuzz/runFuzzTest.sh                   |  2 +-
 4 files changed, 14 insertions(+), 18 deletions(-)

diff --git a/codebuild/spec/buildspec_fuzz.yml b/codebuild/spec/buildspec_fuzz.yml
index 182238809a4..55faa40afdc 100644
--- a/codebuild/spec/buildspec_fuzz.yml
+++ b/codebuild/spec/buildspec_fuzz.yml
@@ -33,4 +33,4 @@ phases:
     on-failure: ABORT
     commands:
       # -L: Restrict tests to names matching the pattern 'fuzz'
-      - cmake --build build/ --target test -- ARGS="-L fuzz --output-on-failure -j (nproc)"
+      - cmake --build build/ --target test -- ARGS="-L fuzz --output-on-failure -j $(nproc)"
diff --git a/codebuild/spec/buildspec_fuzz_scheduled.yml b/codebuild/spec/buildspec_fuzz_scheduled.yml
index 5f26166a943..a8e55d59a5b 100644
--- a/codebuild/spec/buildspec_fuzz_scheduled.yml
+++ b/codebuild/spec/buildspec_fuzz_scheduled.yml
@@ -14,11 +14,6 @@
 version: 0.2
 
 env:
-  compute-type:
-    - BUILD_GENERAL1_LARGE
-  image:
-    - 024603541914.dkr.ecr.us-west-2.amazonaws.com/docker:ubuntu22codebuild
-  privileged-mode: true
   variables:
     CC: "/usr/bin/clang"
     CXX: "/usr/bin/clang++"
@@ -34,12 +29,11 @@ phases:
   build:
     on-failure: ABORT
     commands:
-      #TODO: change fuzz timeout back to 27000
       - |
         cmake . -Bbuild \
         -DCMAKE_PREFIX_PATH=/usr/local/$S2N_LIBCRYPTO \
         -DS2N_FUZZ_TEST=on \
-        -DFUZZ_TIMEOUT_SEC=60 \
+        -DFUZZ_TIMEOUT_SEC=27000 \
         -DCOVERAGE=on \
         -DBUILD_SHARED_LIBS=ON
       - cmake --build ./build -- -j $(nproc)
@@ -47,13 +41,12 @@ phases:
     on-failure: ABORT
     commands:
       # -L: Restrict tests to labels matching the pattern 'fuzz'
-      # -R: Run the single fuzz test defined in ${FUZZ_TESTS}
       # --timeout: override ctest's default timeout of 1500
-      - cmake --build build/ --target test -- ARGS="-L fuzz -R ${FUZZ_TESTS} --output-on-failure -j $(nproc) --timeout 28800"
+      - cmake --build build/ --target test -- ARGS="-L fuzz --output-on-failure -j $(nproc) --timeout 28800"
       - ./tests/fuzz/calcTotalCov.sh
 
 artifacts:
-# upload all files in the coverage_report directory
-files:
-  - '**/*'
-base-directory: fuzz_coverage_report
+  # upload all files in the fuzz_coverage_report directory
+  files:
+    - '**/*'
+  base-directory: fuzz_coverage_report
diff --git a/tests/fuzz/calcTotalCov.sh b/tests/fuzz/calcTotalCov.sh
index 66279cd4e1c..3d1ea02ef3c 100755
--- a/tests/fuzz/calcTotalCov.sh
+++ b/tests/fuzz/calcTotalCov.sh
@@ -26,7 +26,7 @@ fi
 
 FUZZCOV_SOURCES="api bin crypto error stuffer tls utils"
 
-printf "Calculating total s2n coverage... "
+printf "Calculating total s2n coverage... \n"
 
 # The llvm-profdata merge command warns that the profraws were created from different binaries (which is true) but
 # works fine for what we care about (the s2n library). Therefore, for user clarity all output is suppressed.
@@ -39,5 +39,8 @@ llvm-cov export -instr-profile=tests/fuzz/profiles/merged_fuzz.profdata build/li
 
 genhtml s2n_fuzz_cov.info --branch-coverage -q -o fuzz_coverage_report
 
-S2N_COV=`grep -Eo '[0-9]*\.[0-9]*\%' s2n_fuzz_coverage.txt | tail -1`
-printf "total s2n coverage from fuzz tests: %s\n" $S2N_COV
+S2N_COV=`grep -Eo '[0-9]*\.[0-9]*\%' s2n_fuzz_coverage.txt | tail -3`
+LINE_COV=$(echo $S2N_COV | cut -d' ' -f1)
+FUNC_COV=$(echo $S2N_COV | cut -d' ' -f2)
+BRANCH_COV=$(echo $S2N_COV | cut -d' ' -f3)
+printf "Coverage Report:\nLine coverage: %s\nFunction coverage: %s\nBranch coverage: %s\n" "$LINE_COV" "$FUNC_COV" "$BRANCH_COV"
diff --git a/tests/fuzz/runFuzzTest.sh b/tests/fuzz/runFuzzTest.sh
index d2d70b71a0f..841dbade7bf 100755
--- a/tests/fuzz/runFuzzTest.sh
+++ b/tests/fuzz/runFuzzTest.sh
@@ -49,7 +49,7 @@ ASAN_OPTIONS+="symbolize=1"
 LSAN_OPTIONS+="log_threads=1"
 UBSAN_OPTIONS+="print_stacktrace=1"
 NUM_CPU_THREADS=$(nproc)
-LIBFUZZER_ARGS+="-timeout=5 -max_len=4096 -print_final_stats=1 -jobs=${NUM_CPU_THREADS} -workers=${NUM_CPU_THREADS} -max_total_time=${FUZZ_TIMEOUT_SEC}"
+LIBFUZZER_ARGS+="-timeout=5 -max_len=4096 -print_final_stats=1 -workers=${NUM_CPU_THREADS} -max_total_time=${FUZZ_TIMEOUT_SEC}"
 
 TEST_SPECIFIC_OVERRIDES="${BUILD_DIR_PATH}/lib/lib${TEST_NAME}_overrides.so"
 GLOBAL_OVERRIDES="${BUILD_DIR_PATH}/lib/libglobal_overrides.so"

From ce73ffa2e0781bc6bcc73d65d603b4cea1ae3e80 Mon Sep 17 00:00:00 2001
From: Jou Ho <jouh@amazon.com>
Date: Wed, 11 Dec 2024 00:57:50 +0000
Subject: [PATCH 03/10] address PR feedbacks - split report generation logic
 from fuzz runner - remove unused output - use less arguments

modify script name

use correct file name

refactor into smaller components

add execute permissions

suppress noisy output

fix path to profraw

fix profraw path

fix profraw gen
---
 CMakeLists.txt                                |  23 ---
 codebuild/bin/fuzz_corpus_download.sh         |  38 +++++
 codebuild/bin/fuzz_corpus_upload.sh           |  30 ++++
 .../bin/fuzz_coverage_report.sh               |  34 +++-
 codebuild/spec/buildspec_fuzz_scheduled.yml   |   7 +-
 tests/fuzz/runFuzzTest.sh                     | 146 +++---------------
 6 files changed, 119 insertions(+), 159 deletions(-)
 create mode 100755 codebuild/bin/fuzz_corpus_download.sh
 create mode 100755 codebuild/bin/fuzz_corpus_upload.sh
 rename tests/fuzz/calcTotalCov.sh => codebuild/bin/fuzz_coverage_report.sh (59%)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b1144ea69d3..a56871fbece 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -667,7 +667,6 @@ if (BUILD_TESTING)
     if(S2N_FUZZ_TEST)
         message(STATUS "Fuzz build enabled")
         set(SCRIPT_PATH "${CMAKE_CURRENT_SOURCE_DIR}/tests/fuzz/runFuzzTest.sh")
-        set(BUILD_DIR_PATH "${CMAKE_CURRENT_SOURCE_DIR}/build")
         file(GLOB FUZZ_TEST_SRCS "${CMAKE_CURRENT_SOURCE_DIR}/tests/fuzz/*.c")
 
         file(GLOB TESTLIB_SRC "tests/testlib/*.c")
@@ -684,24 +683,6 @@ if (BUILD_TESTING)
             set(FUZZ_TIMEOUT_SEC 60)
         endif()
 
-        if(DEFINED ENV{CORPUS_UPLOAD_LOC})
-            set(CORPUS_UPLOAD_LOC $ENV{CORPUS_UPLOAD_LOC})
-        else()
-            set(CORPUS_UPLOAD_LOC "none")
-        endif()
-
-        if(DEFINED ENV{ARTIFACT_UPLOAD_LOC})
-            set(ARTIFACT_UPLOAD_LOC $ENV{ARTIFACT_UPLOAD_LOC})
-        else()
-            set(ARTIFACT_UPLOAD_LOC "none")
-        endif()
-
-        if(COVERAGE)
-            set(FUZZ_COVERAGE "true")
-        else()
-            set(FUZZ_COVERAGE "false")
-        endif()
-
         # Build LD_PRELOAD shared libraries
         file(GLOB LIBRARY_SRCS "${CMAKE_CURRENT_SOURCE_DIR}/tests/fuzz/LD_PRELOAD/*.c")
         foreach(SRC ${LIBRARY_SRCS})
@@ -735,10 +716,6 @@ if (BUILD_TESTING)
                     bash ${SCRIPT_PATH} 
                     ${TEST_NAME} 
                     ${FUZZ_TIMEOUT_SEC} 
-                    ${BUILD_DIR_PATH} 
-                    ${CORPUS_UPLOAD_LOC} 
-                    ${ARTIFACT_UPLOAD_LOC}
-                    ${FUZZ_COVERAGE}
                     ${CMAKE_CURRENT_SOURCE_DIR}
                 WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests/fuzz
             )
diff --git a/codebuild/bin/fuzz_corpus_download.sh b/codebuild/bin/fuzz_corpus_download.sh
new file mode 100755
index 00000000000..c38612edc77
--- /dev/null
+++ b/codebuild/bin/fuzz_corpus_download.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#  http://aws.amazon.com/apache2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+
+# Clean the environment before copying corpuses from the S3 bucket.
+# The LD variables may interfere with certificate validation when communicating with AWS S3.
+unset LD_PRELOAD
+unset LD_LIBRARY_PATH
+
+for FUZZ_TEST in tests/fuzz/*.c; do
+    # extract file name without extension
+    TEST_NAME=$(basename "$FUZZ_TEST")
+    TEST_NAME="${TEST_NAME%.*}"
+    
+    # temp corpus folder to store downloaded corpus files
+    TEMP_CORPUS_DIR="./tests/fuzz/temp_corpus_${TEST_NAME}"
+
+    # Check if corpus.zip exists in the specified S3 location.
+    # `> /dev/null 2>&1` redirects output to /dev/null.
+    # If the file is not found, `aws s3 ls` returns a non-zero exit code.
+    if aws s3 ls "s3://s2n-tls-fuzz-corpus/${TEST_NAME}/corpus.zip" > /dev/null 2>&1; then
+        aws s3 cp "s3://s2n-tls-fuzz-corpus/${TEST_NAME}/corpus.zip" "${TEMP_CORPUS_DIR}/corpus.zip"
+        unzip -o "${TEMP_CORPUS_DIR}/corpus.zip" -d "${TEMP_CORPUS_DIR}" > /dev/null 2>&1
+    else
+        printf "corpus.zip not found for ${TEST_NAME}"
+    fi
+done
diff --git a/codebuild/bin/fuzz_corpus_upload.sh b/codebuild/bin/fuzz_corpus_upload.sh
new file mode 100755
index 00000000000..17dd4033b38
--- /dev/null
+++ b/codebuild/bin/fuzz_corpus_upload.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#  http://aws.amazon.com/apache2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+
+# Clean the environment before copying corpuses to the S3 bucket.
+# The LD variables may interfere with certificate validation when communicating with AWS S3.
+unset LD_PRELOAD
+unset LD_LIBRARY_PATH
+
+for FUZZ_TEST in tests/fuzz/*.c; do
+    # extract file name without extension
+    TEST_NAME=$(basename "$FUZZ_TEST")
+    TEST_NAME="${TEST_NAME%.*}"
+    
+    # Store generated corpus files in the S3 bucket.
+    zip -r ./tests/fuzz/corpus/${TEST_NAME}.zip ./tests/fuzz/corpus/${TEST_NAME}/ > /dev/null 2>&1
+    aws s3 cp ./tests/fuzz/corpus/${TEST_NAME}.zip s3://s2n-tls-fuzz-corpus/${TEST_NAME}/corpus.zip
+done
+
diff --git a/tests/fuzz/calcTotalCov.sh b/codebuild/bin/fuzz_coverage_report.sh
similarity index 59%
rename from tests/fuzz/calcTotalCov.sh
rename to codebuild/bin/fuzz_coverage_report.sh
index 3d1ea02ef3c..484a9c9db87 100755
--- a/tests/fuzz/calcTotalCov.sh
+++ b/codebuild/bin/fuzz_coverage_report.sh
@@ -24,8 +24,36 @@ if [ "$#" -ne "0" ]; then
     usage
 fi
 
+FUZZ_TEST_DIR="tests/fuzz"
 FUZZCOV_SOURCES="api bin crypto error stuffer tls utils"
 
+# generate coverage report for each fuzz test
+printf "Generating coverage reports... \n"
+
+mkdir -p coverage/fuzz
+for FUZZ_TEST in "$FUZZ_TEST_DIR"/*.c; do
+    # extract file name without extension
+    TEST_NAME=$(basename "$FUZZ_TEST")
+    TEST_NAME="${TEST_NAME%.*}"
+
+    llvm-profdata merge \
+        -sparse tests/fuzz/profiles/${TEST_NAME}/*.profraw \
+        -o tests/fuzz/profiles/${TEST_NAME}/${TEST_NAME}.profdata
+
+    llvm-cov report \
+        -instr-profile=tests/fuzz/profiles/${TEST_NAME}/${TEST_NAME}.profdata build/lib/libs2n.so ${FUZZCOV_SOURCES} \
+        -show-functions \
+        > coverage/fuzz/${TEST_NAME}_cov.txt
+
+    llvm-cov export \
+        -instr-profile=tests/fuzz/profiles/${TEST_NAME}/${TEST_NAME}.profdata build/lib/libs2n.so ${FUZZCOV_SOURCES} \
+        -format=lcov \
+        > coverage/fuzz/${TEST_NAME}_cov.info
+
+    genhtml -q -o coverage/html/${TEST_NAME} coverage/fuzz/${TEST_NAME}_cov.info > /dev/null 2>&1
+done
+
+# merge all coverage reports into a single report that shows total s2n coverage
 printf "Calculating total s2n coverage... \n"
 
 # The llvm-profdata merge command warns that the profraws were created from different binaries (which is true) but
@@ -38,9 +66,3 @@ llvm-cov report -instr-profile=tests/fuzz/profiles/merged_fuzz.profdata build/li
 llvm-cov export -instr-profile=tests/fuzz/profiles/merged_fuzz.profdata build/lib/libs2n.so ${FUZZCOV_SOURCES} -format=lcov > s2n_fuzz_cov.info
 
 genhtml s2n_fuzz_cov.info --branch-coverage -q -o fuzz_coverage_report
-
-S2N_COV=`grep -Eo '[0-9]*\.[0-9]*\%' s2n_fuzz_coverage.txt | tail -3`
-LINE_COV=$(echo $S2N_COV | cut -d' ' -f1)
-FUNC_COV=$(echo $S2N_COV | cut -d' ' -f2)
-BRANCH_COV=$(echo $S2N_COV | cut -d' ' -f3)
-printf "Coverage Report:\nLine coverage: %s\nFunction coverage: %s\nBranch coverage: %s\n" "$LINE_COV" "$FUNC_COV" "$BRANCH_COV"
diff --git a/codebuild/spec/buildspec_fuzz_scheduled.yml b/codebuild/spec/buildspec_fuzz_scheduled.yml
index a8e55d59a5b..dc661c49882 100644
--- a/codebuild/spec/buildspec_fuzz_scheduled.yml
+++ b/codebuild/spec/buildspec_fuzz_scheduled.yml
@@ -33,17 +33,18 @@ phases:
         cmake . -Bbuild \
         -DCMAKE_PREFIX_PATH=/usr/local/$S2N_LIBCRYPTO \
         -DS2N_FUZZ_TEST=on \
-        -DFUZZ_TIMEOUT_SEC=27000 \
         -DCOVERAGE=on \
-        -DBUILD_SHARED_LIBS=ON
+        -DBUILD_SHARED_LIBS=on
       - cmake --build ./build -- -j $(nproc)
   post_build:
     on-failure: ABORT
     commands:
+      - ./codebuild/bin/fuzz_corpus_download.sh
       # -L: Restrict tests to labels matching the pattern 'fuzz'
       # --timeout: override ctest's default timeout of 1500
       - cmake --build build/ --target test -- ARGS="-L fuzz --output-on-failure -j $(nproc) --timeout 28800"
-      - ./tests/fuzz/calcTotalCov.sh
+      - ./codebuild/bin/fuzz_corpus_upload.sh
+      - ./codebuild/bin/fuzz_coverage_report.sh
 
 artifacts:
   # upload all files in the fuzz_coverage_report directory
diff --git a/tests/fuzz/runFuzzTest.sh b/tests/fuzz/runFuzzTest.sh
index 841dbade7bf..5f5fa81d5de 100755
--- a/tests/fuzz/runFuzzTest.sh
+++ b/tests/fuzz/runFuzzTest.sh
@@ -18,21 +18,17 @@
 set -e
 
 usage() {
-    echo "Usage: runFuzzTest.sh TEST_NAME FUZZ_TIMEOUT_SEC BUILD_DIR_PATH CORPUS_UPLOAD_LOC ARTIFACT_UPLOAD_LOC FUZZ_COVERAGE"
+    echo "Usage: runFuzzTest.sh TEST_NAME FUZZ_TIMEOUT_SEC S2N_ROOT"
     exit 1
 }
 
-if [ "$#" -ne "7" ]; then
+if [ "$#" -ne "3" ]; then
     usage
 fi
 
 TEST_NAME=$1
 FUZZ_TIMEOUT_SEC=$2
-BUILD_DIR_PATH=$3
-CORPUS_UPLOAD_LOC=$4
-ARTIFACT_UPLOAD_LOC=$5
-FUZZ_COVERAGE=$6
-S2N_ROOT=$7
+S2N_ROOT=$3
 
 MIN_TEST_PER_SEC="1000"
 MIN_FEATURES_COVERED="100"
@@ -48,11 +44,10 @@ fi
 ASAN_OPTIONS+="symbolize=1"
 LSAN_OPTIONS+="log_threads=1"
 UBSAN_OPTIONS+="print_stacktrace=1"
-NUM_CPU_THREADS=$(nproc)
-LIBFUZZER_ARGS+="-timeout=5 -max_len=4096 -print_final_stats=1 -workers=${NUM_CPU_THREADS} -max_total_time=${FUZZ_TIMEOUT_SEC}"
+LIBFUZZER_ARGS+="-timeout=5 -max_len=4096 -print_final_stats=1 -max_total_time=${FUZZ_TIMEOUT_SEC}"
 
-TEST_SPECIFIC_OVERRIDES="${BUILD_DIR_PATH}/lib/lib${TEST_NAME}_overrides.so"
-GLOBAL_OVERRIDES="${BUILD_DIR_PATH}/lib/libglobal_overrides.so"
+TEST_SPECIFIC_OVERRIDES="${S2N_ROOT}/build/lib/lib${TEST_NAME}_overrides.so"
+GLOBAL_OVERRIDES="${S2N_ROOT}/build/lib/libglobal_overrides.so"
 
 FUZZCOV_SOURCES="${S2N_ROOT}/api ${S2N_ROOT}/bin ${S2N_ROOT}/crypto ${S2N_ROOT}/error ${S2N_ROOT}/stuffer ${S2N_ROOT}/tls ${S2N_ROOT}/utils"
 
@@ -82,41 +77,14 @@ ACTUAL_TEST_FAILURE=0
 
 # Copy existing Corpus to a temp directory so that new inputs from fuzz tests runs will add new inputs to the temp directory.
 # This allows us to minimize new inputs before merging to the original corpus directory.
-# If s3 directory is specified, use corpuses stored in S3 bucket instead.
-TEMP_CORPUS_DIR="$(mktemp -d)"
-if [ "$CORPUS_UPLOAD_LOC" != "none" ]; then
-    (
-        # Clean the environment before copying corpuses from the S3 bucket.
-        # The LD variables interferes with certificate validation when communicating with AWS S3.
-        unset LD_PRELOAD
-        unset LD_LIBRARY_PATH
-
-        # Check if corpus.zip exists in the specified S3 location.
-        # `> /dev/null 2>&1` redirects output to /dev/null.
-        # If the file is not found, `aws s3 ls` returns a non-zero exit code.
-        if aws s3 ls "${CORPUS_UPLOAD_LOC}/${TEST_NAME}/corpus.zip" > /dev/null 2>&1; then
-            printf "corpus.zip found, downloading from S3 bucket and unzipping...\n"
-            aws s3 cp "${CORPUS_UPLOAD_LOC}/${TEST_NAME}/corpus.zip" "${TEMP_CORPUS_DIR}/corpus.zip"
-            unzip -o "${TEMP_CORPUS_DIR}/corpus.zip" -d "${TEMP_CORPUS_DIR}"
-        fi
-    )
-else
-    cp -r ./corpus/${TEST_NAME}/. "${TEMP_CORPUS_DIR}"
-fi
+TEMP_CORPUS_DIR="temp_corpus_${TEST_NAME}"
+cp -r ./corpus/${TEST_NAME}/. "${TEMP_CORPUS_DIR}"
 
-# Setup and clean profile structure if FUZZ_COVERAGE is enabled, otherwise run as normal
-if [[ "$FUZZ_COVERAGE" == "true" ]]; then
-    mkdir -p "./profiles/${TEST_NAME}"
-    rm -f ./profiles/${TEST_NAME}/*.profraw
-    LLVM_PROFILE_FILE="./profiles/${TEST_NAME}/${TEST_NAME}.%p.profraw" \
-    env LD_PRELOAD="$LD_PRELOAD_" \
-    ${BUILD_DIR_PATH}/bin/${TEST_NAME} ${LIBFUZZER_ARGS} ${TEMP_CORPUS_DIR} \
-    > ${TEST_NAME}_output.txt 2>&1 || ACTUAL_TEST_FAILURE=1
-else
-    env LD_PRELOAD="$LD_PRELOAD_" \
-    ${BUILD_DIR_PATH}/bin/${TEST_NAME} ${LIBFUZZER_ARGS} ${TEMP_CORPUS_DIR} \
-    > ${TEST_NAME}_output.txt 2>&1 || ACTUAL_TEST_FAILURE=1
-fi
+# Run fuzz test executable and store results to an output file
+env LD_PRELOAD="$LD_PRELOAD_" \
+LLVM_PROFILE_FILE="./profiles/${TEST_NAME}/${TEST_NAME}.%p.profraw" \
+${S2N_ROOT}/build/bin/${TEST_NAME} ${LIBFUZZER_ARGS} ${TEMP_CORPUS_DIR} \
+> ${TEST_NAME}_output.txt 2>&1 || ACTUAL_TEST_FAILURE=1
 
 TEST_INFO=$(
     grep -o "stat::number_of_executed_units: [0-9]*" ${TEST_NAME}_output.txt | \
@@ -124,68 +92,19 @@ TEST_INFO=$(
 )
 TESTS_PER_SEC=$(echo "$TEST_INFO" | cut -d ' ' -f 3)
 FEATURE_COVERAGE=`grep -o "ft: [0-9]*" ${TEST_NAME}_output.txt | awk '{print $2}' | sort | tail -1`
-TARGET_FUNCS=''
-declare -i TARGET_TOTAL=0
-declare -i TARGET_COV=0
-
-# Outputs fuzz coverage results if the FUZZ_COVERAGE environment variable is set
-# Coverage is overlayed on source code in ${TEST_NAME}_cov.html, and coverage statistics are available in ${TEST_NAME}_cov.txt
-# If using LLVM version 9 or greater, coverage is output in LCOV format instead of HTML
-# All files are stored in the s2n coverage directory
-if [[ "$FUZZ_COVERAGE" == "true" ]]; then
-    mkdir -p coverage/fuzz
-    llvm-profdata merge -sparse ./profiles/${TEST_NAME}/*.profraw -o ./profiles/${TEST_NAME}/${TEST_NAME}.profdata
-    llvm-cov report -instr-profile=./profiles/${TEST_NAME}/${TEST_NAME}.profdata ${BUILD_DIR_PATH}/lib/libs2n.so ${FUZZCOV_SOURCES} -show-functions > coverage/fuzz/${TEST_NAME}_cov.txt
-
-    # Use LCOV format instead of HTML if the LLVM version we're using supports it
-    if [[ $(grep -Eo "[0-9]*" <<< `llvm-cov --version` | head -1) -gt 8 ]]; then
-        llvm-cov export -instr-profile=./profiles/${TEST_NAME}/${TEST_NAME}.profdata ${BUILD_DIR_PATH}/lib/libs2n.so ${FUZZCOV_SOURCES} -format=lcov > coverage/fuzz/${TEST_NAME}_cov.info
-        genhtml -q -o coverage/html/${TEST_NAME} coverage/fuzz/${TEST_NAME}_cov.info
-    else
-        llvm-cov show -instr-profile=./profiles/${TEST_NAME}/${TEST_NAME}.profdata ${BUILD_DIR_PATH}/lib/libs2n.so ${FUZZCOV_SOURCES} -use-color -format=html > coverage/fuzz/${TEST_NAME}_cov.html
-    fi
-
-    # Extract target functions from test source
-    TARGET_FUNCS=`grep -Pzo "(?<=/\* Target Functions: )[\w\s]*" ${TEST_NAME}.c | tr -d "\0"`
-
-    # Find line coverage statistics for target functions
-    if [[ ! -z "$TARGET_FUNCS" ]];
-    then
-        for TARGET in ${TARGET_FUNCS}
-        do
-            TARGET_TOTAL+=`sed -n "s/^.*${TARGET} .*% *\([0-9]*\) .*$/\1/p" coverage/fuzz/${TEST_NAME}_cov.txt`
-            TARGET_COV+=`sed -n "s/^.*${TARGET} .*% *[0-9]* *\([0-9]*\) .*$/\1/p" coverage/fuzz/${TEST_NAME}_cov.txt`
-        done
-    fi
-fi
 
 if [ $ACTUAL_TEST_FAILURE == $EXPECTED_TEST_FAILURE ];
 then
-    printf "\033[32;1mPASSED\033[0m %s" "$TEST_INFO"
-
-    # Output target function coverage percentage if target functions are defined and fuzzing coverage is enabled
-    # Otherwise, print number of features covered
-    if [[ "$FUZZ_COVERAGE" == "true" && ! -z "$TARGET_FUNCS" && "$EXPECTED_TEST_FAILURE" != 1 && "$TARGET_TOTAL" != 0 ]];
-    then
-        printf ", %6.2f%% target coverage" "$(( 10000 * ($TARGET_TOTAL - $TARGET_COV) / $TARGET_TOTAL ))e-2"
-    else
-        printf ", %5d features covered" $FEATURE_COVERAGE
-    fi
-
     if [ $EXPECTED_TEST_FAILURE == 1 ];
     then
         # Clean up LibFuzzer corpus files if the test is negative.
-        printf "\n"
         rm -f leak-* crash-*
     else
         # TEMP_CORPUS_DIR may contain many new inputs that only covers a small set of new branches. 
         # Instead of copying all new inputs to the corpus directory,  only copy back minimum number of new inputs that reach new branches.
-        ${BUILD_DIR_PATH}/bin/${TEST_NAME} -merge=1 "./corpus/${TEST_NAME}" "${TEMP_CORPUS_DIR}" \
-        > ${TEST_NAME}_results.txt 2>&1
-
-        # Print number of new files and branches found in new Inputs (if any)
-        RESULTS=`grep -Eo "[0-9]+ new files .*$" ${TEST_NAME}_results.txt | tail -1`
-        printf ", ${RESULTS}\n"
+        ${S2N_ROOT}/build/bin/${TEST_NAME} \
+            -merge=1 "./corpus/${TEST_NAME}" "${TEMP_CORPUS_DIR}" \
+            > ${TEST_NAME}_results.txt 2>&1
 
         if [ "$TESTS_PER_SEC" -lt $MIN_TEST_PER_SEC ]; then
             printf "\033[33;1mWARNING!\033[0m ${TEST_NAME} is only ${TESTS_PER_SEC} tests/sec, which is below ${MIN_TEST_PER_SEC}/sec! Fuzz tests are more effective at higher rates.\n\n"
@@ -201,39 +120,12 @@ then
             printf "\033[31;1mERROR!\033[0m ${TEST_NAME} only covers ${FEATURE_COVERAGE} features, which is below ${MIN_FEATURES_COVERED}! This may be due to missing corpus files or a bug.\n"
             exit -1;
         fi
-
-        # Store generated corpus files in the S3 bucket.
-        unset LD_PRELOAD
-        unset LD_LIBRARY_PATH
-        if [ "$CORPUS_UPLOAD_LOC" != "none" ]; then
-            printf "Zipping corpus files...\n"
-            zip -r ./corpus/${TEST_NAME}.zip ./corpus/${TEST_NAME}/
-            
-            printf "Uploading zipped corpus file to S3 bucket...\n"
-            aws s3 cp ./corpus/${TEST_NAME}.zip $CORPUS_UPLOAD_LOC/${TEST_NAME}/corpus.zip
-        fi
     fi
-
 else
     cat ${TEST_NAME}_output.txt
     printf "\033[31;1mFAILED\033[0m %s, %6d features covered\n" "$TEST_INFO" $FEATURE_COVERAGE
-    
-    # Store corpus to S3 to be used for debugging if the test fails
-    unset LD_PRELOAD
-    unset LD_LIBRARY_PATH
-    if [ "$CORPUS_UPLOAD_LOC" != "none" ]; then
-        printf "Zipping corpus files...\n"
-        zip -r ./corpus/${TEST_NAME}.zip ./corpus/${TEST_NAME}/
-        
-        printf "Uploading zipped corpus file to S3 bucket...\n"
-        aws s3 cp ./corpus/${TEST_NAME}.zip $CORPUS_UPLOAD_LOC/${TEST_NAME}/corpus_$(date +%Y-%m-%d-%T).zip
-    fi
-
-    # Store generated output files in the S3 bucket.
-    if [ "$ARTIFACT_UPLOAD_LOC" != "none" ]; then
-        printf "Uploading output files to S3 bucket...\n"
-        aws s3 cp ./${TEST_NAME}_output.txt ${ARTIFACT_UPLOAD_LOC}/${TEST_NAME}/output_$(date +%Y-%m-%d-%T).txt
-        aws s3 cp ./${TEST_NAME}_results.txt ${ARTIFACT_UPLOAD_LOC}/${TEST_NAME}/results_$(date +%Y-%m-%d-%T).txt
-    fi
+    # Store generated output files in the S3 bucket for debugging.
+    aws s3 cp ./tests/fuzz/${TEST_NAME}_output.txt ${ARTIFACT_UPLOAD_LOC}/${TEST_NAME}/output_$(date +%Y-%m-%d-%T).txt
+    aws s3 cp ./tests/fuzz/${TEST_NAME}_results.txt ${ARTIFACT_UPLOAD_LOC}/${TEST_NAME}/results_$(date +%Y-%m-%d-%T).txt
     exit -1
 fi

From 6352fe429596f40875c6baf5d70de6e921325fd2 Mon Sep 17 00:00:00 2001
From: Jou Ho <jouh@amazon.com>
Date: Thu, 12 Dec 2024 01:38:34 +0000
Subject: [PATCH 04/10] document report gen steps

---
 codebuild/bin/fuzz_coverage_report.sh | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/codebuild/bin/fuzz_coverage_report.sh b/codebuild/bin/fuzz_coverage_report.sh
index 484a9c9db87..5b039e14ae9 100755
--- a/codebuild/bin/fuzz_coverage_report.sh
+++ b/codebuild/bin/fuzz_coverage_report.sh
@@ -36,33 +36,40 @@ for FUZZ_TEST in "$FUZZ_TEST_DIR"/*.c; do
     TEST_NAME=$(basename "$FUZZ_TEST")
     TEST_NAME="${TEST_NAME%.*}"
 
+    # merge multiple .profraw files into a single .profdata file
     llvm-profdata merge \
         -sparse tests/fuzz/profiles/${TEST_NAME}/*.profraw \
         -o tests/fuzz/profiles/${TEST_NAME}/${TEST_NAME}.profdata
 
+    # generate a coverage report in text format
     llvm-cov report \
         -instr-profile=tests/fuzz/profiles/${TEST_NAME}/${TEST_NAME}.profdata build/lib/libs2n.so ${FUZZCOV_SOURCES} \
         -show-functions \
         > coverage/fuzz/${TEST_NAME}_cov.txt
 
+    # exports coverage data in LCOV format
     llvm-cov export \
         -instr-profile=tests/fuzz/profiles/${TEST_NAME}/${TEST_NAME}.profdata build/lib/libs2n.so ${FUZZCOV_SOURCES} \
         -format=lcov \
         > coverage/fuzz/${TEST_NAME}_cov.info
 
+    # convert to HTML format
     genhtml -q -o coverage/html/${TEST_NAME} coverage/fuzz/${TEST_NAME}_cov.info > /dev/null 2>&1
 done
 
 # merge all coverage reports into a single report that shows total s2n coverage
 printf "Calculating total s2n coverage... \n"
+llvm-profdata merge \
+    -sparse tests/fuzz/profiles/*/*.profdata \
+    -o tests/fuzz/profiles/merged_fuzz.profdata
 
-# The llvm-profdata merge command warns that the profraws were created from different binaries (which is true) but
-# works fine for what we care about (the s2n library). Therefore, for user clarity all output is suppressed.
-llvm-profdata merge -sparse tests/fuzz/profiles/*/*.profdata -o tests/fuzz/profiles/merged_fuzz.profdata > /dev/null 2>&1
-
-llvm-cov report -instr-profile=tests/fuzz/profiles/merged_fuzz.profdata build/lib/libs2n.so ${FUZZCOV_SOURCES} > s2n_fuzz_coverage.txt
-
-# convert coverage information to html format
-llvm-cov export -instr-profile=tests/fuzz/profiles/merged_fuzz.profdata build/lib/libs2n.so ${FUZZCOV_SOURCES} -format=lcov > s2n_fuzz_cov.info
+llvm-cov report \ 
+    -instr-profile=tests/fuzz/profiles/merged_fuzz.profdata build/lib/libs2n.so ${FUZZCOV_SOURCES} \
+    > s2n_fuzz_coverage.txt
 
+llvm-cov export \ 
+    -instr-profile=tests/fuzz/profiles/merged_fuzz.profdata build/lib/libs2n.so ${FUZZCOV_SOURCES} \ 
+    -format=lcov \ 
+    > s2n_fuzz_cov.info
+    
 genhtml s2n_fuzz_cov.info --branch-coverage -q -o fuzz_coverage_report

From daab18d337cd9b898a0e04efb82f76cd40a21931 Mon Sep 17 00:00:00 2001
From: Jou Ho <jouh@amazon.com>
Date: Thu, 12 Dec 2024 18:23:19 +0000
Subject: [PATCH 05/10] fix bash script spacings & change output dir

---
 codebuild/bin/fuzz_coverage_report.sh       | 10 +++++-----
 codebuild/spec/buildspec_fuzz_scheduled.yml |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/codebuild/bin/fuzz_coverage_report.sh b/codebuild/bin/fuzz_coverage_report.sh
index 5b039e14ae9..6e2a4afcf23 100755
--- a/codebuild/bin/fuzz_coverage_report.sh
+++ b/codebuild/bin/fuzz_coverage_report.sh
@@ -63,13 +63,13 @@ llvm-profdata merge \
     -sparse tests/fuzz/profiles/*/*.profdata \
     -o tests/fuzz/profiles/merged_fuzz.profdata
 
-llvm-cov report \ 
+llvm-cov report \
     -instr-profile=tests/fuzz/profiles/merged_fuzz.profdata build/lib/libs2n.so ${FUZZCOV_SOURCES} \
     > s2n_fuzz_coverage.txt
 
-llvm-cov export \ 
-    -instr-profile=tests/fuzz/profiles/merged_fuzz.profdata build/lib/libs2n.so ${FUZZCOV_SOURCES} \ 
-    -format=lcov \ 
+llvm-cov export \
+    -instr-profile=tests/fuzz/profiles/merged_fuzz.profdata build/lib/libs2n.so ${FUZZCOV_SOURCES} \
+    -format=lcov \
     > s2n_fuzz_cov.info
     
-genhtml s2n_fuzz_cov.info --branch-coverage -q -o fuzz_coverage_report
+genhtml s2n_fuzz_cov.info --branch-coverage -q -o coverage/fuzz/total_fuzz_coverage
diff --git a/codebuild/spec/buildspec_fuzz_scheduled.yml b/codebuild/spec/buildspec_fuzz_scheduled.yml
index dc661c49882..9b134638b6c 100644
--- a/codebuild/spec/buildspec_fuzz_scheduled.yml
+++ b/codebuild/spec/buildspec_fuzz_scheduled.yml
@@ -50,4 +50,4 @@ artifacts:
   # upload all files in the fuzz_coverage_report directory
   files:
     - '**/*'
-  base-directory: fuzz_coverage_report
+  base-directory: coverage/fuzz/fuzz_coverage_report

From e1b3a1d1fd013978d0e1f02dfd159eb8b9dd23a9 Mon Sep 17 00:00:00 2001
From: Jou Ho <jouh@amazon.com>
Date: Thu, 12 Dec 2024 22:11:15 +0000
Subject: [PATCH 06/10] update usage documentation

---
 tests/fuzz/Readme.md | 40 ++++++++++++++++++++++++++++++----------
 1 file changed, 30 insertions(+), 10 deletions(-)

diff --git a/tests/fuzz/Readme.md b/tests/fuzz/Readme.md
index 28e4bb2da3a..eaf4ff2b2e3 100644
--- a/tests/fuzz/Readme.md
+++ b/tests/fuzz/Readme.md
@@ -29,23 +29,43 @@ cmake --build build/ --target test -- ARGS="-L fuzz -R s2n_client_fuzz_test --ou
     2. If the test ends with `*_negative_test.c` the test is expected to fail in some way or return a non-zero integer (hereafter referred to as a "Negative test").
 2. Strive to be deterministic (Eg. shouldn't depend on the time or on the output of a RNG). Each test should either always pass if a Positive Test, or always fail if a Negative Test.
 3. If a Positive Fuzz test, it should have a non-empty corpus directory with inputs that have a relatively high branch coverage.
-4. Have a function `int s2n_fuzz_init(int *argc, char **argv[])` that will perform any initialization that will be run only once at startup.
-5. Have a function `int s2n_fuzz_test(const uint8_t *buf, size_t len)` that will pass `buf` to one of s2n's API's
-5. Optionally add a function `void s2n_fuzz_cleanup()` which cleans up any global state.
-6. Call `S2N_FUZZ_TARGET(s2n_fuzz_init, s2n_fuzz_test, s2n_fuzz_cleanup)` at the bottom of the test to initialize the fuzz target
+4. If a Positive Fuzz test, define target functions for the test by adding following lines to your test below the copyright notice:
+> /* Target Functions: function1 function2 function3 */
+5. Have a function `int s2n_fuzz_init(int *argc, char **argv[])` that will perform any initialization that will be run only once at startup.
+6. Have a function `int s2n_fuzz_test(const uint8_t *buf, size_t len)` that will pass `buf` to one of s2n's API's
+7. Optionally add a function `void s2n_fuzz_cleanup()` which cleans up any global state.
+8. Call `S2N_FUZZ_TARGET(s2n_fuzz_init, s2n_fuzz_test, s2n_fuzz_cleanup)` at the bottom of the test to initialize the fuzz target
 
 ## Fuzz Test Coverage
-To generate coverage reports for fuzz tests, simply set the FUZZ_COVERAGE environment variable to any non-null value and run `make fuzz`. This will report the target function coverage and overall S2N coverage when running the tests. In order to define target functions for a fuzz test, simply add the following line to your fuzz test below the copyright notice:
+To generate coverage reports for fuzz tests, s2n-tls needs to be compiled with the following options:
+```
+cmake . -Bbuild \
+-DCMAKE_PREFIX_PATH=/usr/local/$S2N_LIBCRYPTO \
+-DS2N_FUZZ_TEST=on \
+-DCOVERAGE=on \
+-DBUILD_SHARED_LIBS=on
 
-> /* Target Functions: function1 function2 function3 */
+cmake --build ./build -- -j $(nproc)
+```
 
-As the tests run, more detailed coverage reports are placed in the following directory:
+If you see an error that says `This project requires clang for coverage support. You are currently using X`, set the C compiler environment variable to use Clang:
+```
+export CC=/usr/bin/clang
+export CXX=/usr/bin/clang++
+```
 
-> s2n/coverage/fuzz
+Next, run fuzz tests. This will generate `.info` files for each fuzz test containing coverage information. 
+```
+cmake --build build/ --target test -- ARGS="-L fuzz --output-on-failure"
+```
 
-Each test outputs an HTML file which displays line by line coverage statistics and a .txt report which gives per-function coverage statistics in human-readable ASCII. After all fuzz tests have ran, a matching pair of coverage reports is generated for the total coverage of S2N by the entire set of tests performed.
+The `.info` files contain raw coverage data.  To convert them into HTML format, run the following script from the root of s2n-tls. This generates HTML files showing line-by-line coverage statistics for each fuzz test, as well as total coverage report for s2n-tls across all tests.
+```
+./codebuild/bin/fuzz_coverage_report.sh
+```
 
-Currently, this option isn't enabled for cmake build. See [#4748](https://github.com/aws/s2n-tls/issues/4748).
+You will see coverage reports placed in the following directory:
+> s2n-tls/tests/fuzz/coverage
 
 ## Fuzz Test Directory Structure
 For a test with name `$TEST_NAME`, its files should be laid out with the following structure:

From 4a4a3229351fc6e3f20caf0ad5c800821a552e08 Mon Sep 17 00:00:00 2001
From: Jou Ho <jouh@amazon.com>
Date: Thu, 12 Dec 2024 22:12:33 +0000
Subject: [PATCH 07/10] minor doc edit

---
 tests/fuzz/Readme.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/fuzz/Readme.md b/tests/fuzz/Readme.md
index eaf4ff2b2e3..f7cd662ed76 100644
--- a/tests/fuzz/Readme.md
+++ b/tests/fuzz/Readme.md
@@ -54,7 +54,7 @@ export CC=/usr/bin/clang
 export CXX=/usr/bin/clang++
 ```
 
-Next, run fuzz tests. This will generate `.info` files for each fuzz test containing coverage information. 
+Next, run fuzz tests. This generates `.info` files for each fuzz test containing coverage information. 
 ```
 cmake --build build/ --target test -- ARGS="-L fuzz --output-on-failure"
 ```

From 6e12ef281cd805a630d0e55bdd5c69cc627d041a Mon Sep 17 00:00:00 2001
From: Jou Ho <jouh@amazon.com>
Date: Thu, 12 Dec 2024 23:02:08 +0000
Subject: [PATCH 08/10] fix base-directory for artifact path

---
 codebuild/spec/buildspec_fuzz_scheduled.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/codebuild/spec/buildspec_fuzz_scheduled.yml b/codebuild/spec/buildspec_fuzz_scheduled.yml
index 9b134638b6c..34c323d6607 100644
--- a/codebuild/spec/buildspec_fuzz_scheduled.yml
+++ b/codebuild/spec/buildspec_fuzz_scheduled.yml
@@ -50,4 +50,4 @@ artifacts:
   # upload all files in the fuzz_coverage_report directory
   files:
     - '**/*'
-  base-directory: coverage/fuzz/fuzz_coverage_report
+  base-directory: coverage/fuzz/total_fuzz_coverage

From be230099e2ff8717561695619fb264bef4d8adaf Mon Sep 17 00:00:00 2001
From: Jou Ho <jouh@amazon.com>
Date: Fri, 13 Dec 2024 19:26:28 +0000
Subject: [PATCH 09/10] add link to coverage page

---
 tests/fuzz/Readme.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/fuzz/Readme.md b/tests/fuzz/Readme.md
index f7cd662ed76..c178f6473ca 100644
--- a/tests/fuzz/Readme.md
+++ b/tests/fuzz/Readme.md
@@ -37,6 +37,8 @@ cmake --build build/ --target test -- ARGS="-L fuzz -R s2n_client_fuzz_test --ou
 8. Call `S2N_FUZZ_TARGET(s2n_fuzz_init, s2n_fuzz_test, s2n_fuzz_cleanup)` at the bottom of the test to initialize the fuzz target
 
 ## Fuzz Test Coverage
+We run fuzz tests daily, with corpus files that are continuously being improved. Current coverage information can be view [here](https://dx1inn44oyl7n.cloudfront.net/fuzz-coverage-report/index.html).
+
 To generate coverage reports for fuzz tests, s2n-tls needs to be compiled with the following options:
 ```
 cmake . -Bbuild \
@@ -82,9 +84,7 @@ For a test with name `$TEST_NAME`, its files should be laid out with the followi
 # Corpus
 A Corpus is a directory of "interesting" inputs that result in a good branch/code coverage. These inputs will be permuted in random ways and checked to see if this permutation results in greater branch coverage or in a failure (Segfault, Memory Leak, Buffer Overflow, Non-zero return code, etc). If the permutation results in greater branch coverage, then it will be added to the Corpus directory. If a Memory leak or a Crash is detected, that file will **not** be added to the corpus for that test, and will instead be written to the current directory (`s2n/tests/fuzz/crash-*` or `s2n/tests/fuzz/leak-*`). These files will be automatically deleted for any Negative Fuzz tests that are expected to crash or leak memory so as to not clutter the directory.
 
-To continuously improve corpus inputs, we have a scheduled job that runs every day for approximately 8 hours. These tests begin with corpus files stored in an S3 bucket. At the end of each run, the existing corpus files are replaced with updated ones, potentially increasing branch coverage over time. This process allows for gradual and automated enhancement of the corpus.
-
-To enable this, two environment variables must be defined: `CORPUS_UPLOAD_LOC` and `ARTIFACT_UPLOAD_LOC`. `CORPUS_UPLOAD_LOC` specifies where corpus files are stored, while `ARTIFACT_UPLOAD_LOC`defines where output logs from fuzzing are saved, which can be used for debugging if a new bug is detected during fuzzing.
+To continuously improve corpus inputs, we have a scheduled job that runs every day for approximately 8 hours. These tests begin with corpus files stored in an S3 bucket. At the end of each run, the existing corpus files are replaced with updated ones, potentially increasing coverage over time. This process allows for gradual and automated enhancement of the corpus.
 
 # LD_PRELOAD
 The `LD_PRELOAD` directory contains function overrides for each Fuzz test that will be used **instead** of the original functions defined elsewhere. These function overrides will only be used during fuzz tests, and will not effect the rest of the s2n codebase when not fuzzing. Using `LD_PRELOAD` instead of C Preprocessor `#ifdef`'s is preferable in the following ways:

From a2e4156e834d7e41b3069a6e5d83f95d0508e2d3 Mon Sep 17 00:00:00 2001
From: Jou Ho <jouh@amazon.com>
Date: Fri, 13 Dec 2024 23:48:32 +0000
Subject: [PATCH 10/10] address PR feedbacks - remove unset LD_PRELOADS - fix
 script name in the usage - use cmake to configure compiler - remove unused
 options

---
 codebuild/bin/fuzz_corpus_download.sh       | 5 -----
 codebuild/bin/fuzz_corpus_upload.sh         | 7 +------
 codebuild/bin/fuzz_coverage_report.sh       | 2 +-
 codebuild/spec/buildspec_fuzz_scheduled.yml | 4 ++--
 tests/fuzz/Readme.md                        | 6 ------
 tests/fuzz/runFuzzTest.sh                   | 3 ---
 6 files changed, 4 insertions(+), 23 deletions(-)

diff --git a/codebuild/bin/fuzz_corpus_download.sh b/codebuild/bin/fuzz_corpus_download.sh
index c38612edc77..0da91abcbf6 100755
--- a/codebuild/bin/fuzz_corpus_download.sh
+++ b/codebuild/bin/fuzz_corpus_download.sh
@@ -13,11 +13,6 @@
 # permissions and limitations under the License.
 #
 
-# Clean the environment before copying corpuses from the S3 bucket.
-# The LD variables may interfere with certificate validation when communicating with AWS S3.
-unset LD_PRELOAD
-unset LD_LIBRARY_PATH
-
 for FUZZ_TEST in tests/fuzz/*.c; do
     # extract file name without extension
     TEST_NAME=$(basename "$FUZZ_TEST")
diff --git a/codebuild/bin/fuzz_corpus_upload.sh b/codebuild/bin/fuzz_corpus_upload.sh
index 17dd4033b38..8100e074166 100755
--- a/codebuild/bin/fuzz_corpus_upload.sh
+++ b/codebuild/bin/fuzz_corpus_upload.sh
@@ -13,17 +13,12 @@
 # permissions and limitations under the License.
 #
 
-# Clean the environment before copying corpuses to the S3 bucket.
-# The LD variables may interfere with certificate validation when communicating with AWS S3.
-unset LD_PRELOAD
-unset LD_LIBRARY_PATH
-
 for FUZZ_TEST in tests/fuzz/*.c; do
     # extract file name without extension
     TEST_NAME=$(basename "$FUZZ_TEST")
     TEST_NAME="${TEST_NAME%.*}"
     
-    # Store generated corpus files in the S3 bucket.
+    # Upload generated corpus files to the S3 bucket.
     zip -r ./tests/fuzz/corpus/${TEST_NAME}.zip ./tests/fuzz/corpus/${TEST_NAME}/ > /dev/null 2>&1
     aws s3 cp ./tests/fuzz/corpus/${TEST_NAME}.zip s3://s2n-tls-fuzz-corpus/${TEST_NAME}/corpus.zip
 done
diff --git a/codebuild/bin/fuzz_coverage_report.sh b/codebuild/bin/fuzz_coverage_report.sh
index 6e2a4afcf23..a35e4e77d4e 100755
--- a/codebuild/bin/fuzz_coverage_report.sh
+++ b/codebuild/bin/fuzz_coverage_report.sh
@@ -16,7 +16,7 @@
 set -e
 
 usage() {
-    echo "Usage: calcTotalCov.sh"
+    echo "Usage: fuzz_coverage_report.sh"
     exit 1
 }
 
diff --git a/codebuild/spec/buildspec_fuzz_scheduled.yml b/codebuild/spec/buildspec_fuzz_scheduled.yml
index 34c323d6607..29c9c8b354e 100644
--- a/codebuild/spec/buildspec_fuzz_scheduled.yml
+++ b/codebuild/spec/buildspec_fuzz_scheduled.yml
@@ -15,9 +15,8 @@ version: 0.2
 
 env:
   variables:
-    CC: "/usr/bin/clang"
-    CXX: "/usr/bin/clang++"
     S2N_LIBCRYPTO: "awslc"
+    COMPILER: clang
 
 phases:
   pre_build:
@@ -32,6 +31,7 @@ phases:
       - |
         cmake . -Bbuild \
         -DCMAKE_PREFIX_PATH=/usr/local/$S2N_LIBCRYPTO \
+        -DCMAKE_C_COMPILER=/usr/bin/$COMPILER \
         -DS2N_FUZZ_TEST=on \
         -DCOVERAGE=on \
         -DBUILD_SHARED_LIBS=on
diff --git a/tests/fuzz/Readme.md b/tests/fuzz/Readme.md
index c178f6473ca..49fe930f39b 100644
--- a/tests/fuzz/Readme.md
+++ b/tests/fuzz/Readme.md
@@ -50,12 +50,6 @@ cmake . -Bbuild \
 cmake --build ./build -- -j $(nproc)
 ```
 
-If you see an error that says `This project requires clang for coverage support. You are currently using X`, set the C compiler environment variable to use Clang:
-```
-export CC=/usr/bin/clang
-export CXX=/usr/bin/clang++
-```
-
 Next, run fuzz tests. This generates `.info` files for each fuzz test containing coverage information. 
 ```
 cmake --build build/ --target test -- ARGS="-L fuzz --output-on-failure"
diff --git a/tests/fuzz/runFuzzTest.sh b/tests/fuzz/runFuzzTest.sh
index 5f5fa81d5de..1a70e797fb3 100755
--- a/tests/fuzz/runFuzzTest.sh
+++ b/tests/fuzz/runFuzzTest.sh
@@ -41,9 +41,6 @@ else
     EXPECTED_TEST_FAILURE=0
 fi
 
-ASAN_OPTIONS+="symbolize=1"
-LSAN_OPTIONS+="log_threads=1"
-UBSAN_OPTIONS+="print_stacktrace=1"
 LIBFUZZER_ARGS+="-timeout=5 -max_len=4096 -print_final_stats=1 -max_total_time=${FUZZ_TIMEOUT_SEC}"
 
 TEST_SPECIFIC_OVERRIDES="${S2N_ROOT}/build/lib/lib${TEST_NAME}_overrides.so"