From c230dc8aca55d11e1de63d028707393d1a0c6bb5 Mon Sep 17 00:00:00 2001 From: abitmore Date: Thu, 2 Jul 2020 15:45:11 -0400 Subject: [PATCH 1/5] Build test cases for code coverage data --- programs/build_helpers/scan_with_sonar_step_1 | 2 +- programs/build_helpers/scan_with_sonar_step_2 | 2 +- programs/build_helpers/scan_with_sonar_step_3 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/programs/build_helpers/scan_with_sonar_step_1 b/programs/build_helpers/scan_with_sonar_step_1 index 7601184665..f5d2075eba 100755 --- a/programs/build_helpers/scan_with_sonar_step_1 +++ b/programs/build_helpers/scan_with_sonar_step_1 @@ -5,7 +5,7 @@ programs/build_helpers/buildstep -s 3500 ccache -s programs/build_helpers/buildstep Prepare 1 "sed -i '/tests/d' libraries/fc/CMakeLists.txt" programs/build_helpers/buildstep cmake 5 "cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS=--coverage -DCMAKE_CXX_FLAGS=--coverage -DBoost_USE_STATIC_LIBS=OFF -DCMAKE_CXX_OUTPUT_EXTENSION_REPLACE=ON ." -programs/build_helpers/buildstep make.programs 1800 "programs/build_helpers/make_with_sonar bw-output -j 2 witness_node cli_wallet js_operation_serializer get_dev_key network_mapper" +programs/build_helpers/buildstep make.programs 1800 "programs/build_helpers/make_with_sonar bw-output -j 2 witness_node cli_wallet js_operation_serializer get_dev_key network_mapper app_test chain_test cli_test" set -o pipefail programs/build_helpers/buildstep prepare.sonar 20 "find libraries/[acdenptuw]*/CMakeFiles/*.dir programs/[cdgjsw]*/CMakeFiles/*.dir -type d -print | while read d; do gcov -o \"\$d\" \"\${d/CMakeFiles*.dir//}\"/*.cpp; done >/dev/null; programs/build_helpers/set_sonar_branch sonar-project.properties" du -hs sonar_cache diff --git a/programs/build_helpers/scan_with_sonar_step_2 b/programs/build_helpers/scan_with_sonar_step_2 index ae3af7bcb0..3c92765749 100755 --- a/programs/build_helpers/scan_with_sonar_step_2 +++ b/programs/build_helpers/scan_with_sonar_step_2 @@ -5,7 +5,7 @@ programs/build_helpers/buildstep -s 3500 ccache -s programs/build_helpers/buildstep Prepare 1 "sed -i '/tests/d' libraries/fc/CMakeLists.txt" programs/build_helpers/buildstep cmake 5 "cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS=--coverage -DCMAKE_CXX_FLAGS=--coverage -DBoost_USE_STATIC_LIBS=OFF -DCMAKE_CXX_OUTPUT_EXTENSION_REPLACE=ON ." -programs/build_helpers/buildstep make.programs 1800 "programs/build_helpers/make_with_sonar bw-output -j 2 witness_node cli_wallet js_operation_serializer get_dev_key network_mapper" +programs/build_helpers/buildstep make.programs 1800 "programs/build_helpers/make_with_sonar bw-output -j 2 witness_node cli_wallet js_operation_serializer get_dev_key network_mapper app_test chain_test cli_test" set -o pipefail programs/build_helpers/buildstep prepare.sonar 20 "find libraries/[acdenptuw]*/CMakeFiles/*.dir programs/[cdgjsw]*/CMakeFiles/*.dir -type d -print | while read d; do gcov -o \"\$d\" \"\${d/CMakeFiles*.dir//}\"/*.cpp; done >/dev/null; programs/build_helpers/set_sonar_branch sonar-project.properties" du -hs sonar_cache diff --git a/programs/build_helpers/scan_with_sonar_step_3 b/programs/build_helpers/scan_with_sonar_step_3 index 2c18dabc9e..d5105e17e0 100755 --- a/programs/build_helpers/scan_with_sonar_step_3 +++ b/programs/build_helpers/scan_with_sonar_step_3 @@ -5,7 +5,7 @@ programs/build_helpers/buildstep -s 3500 ccache -s programs/build_helpers/buildstep Prepare 1 "sed -i '/tests/d' libraries/fc/CMakeLists.txt" programs/build_helpers/buildstep cmake 5 "cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS=--coverage -DCMAKE_CXX_FLAGS=--coverage -DBoost_USE_STATIC_LIBS=OFF -DCMAKE_CXX_OUTPUT_EXTENSION_REPLACE=ON ." -programs/build_helpers/buildstep make.programs 1800 "programs/build_helpers/make_with_sonar bw-output -j 2 witness_node cli_wallet js_operation_serializer get_dev_key network_mapper" +programs/build_helpers/buildstep make.programs 1800 "programs/build_helpers/make_with_sonar bw-output -j 2 witness_node cli_wallet js_operation_serializer get_dev_key network_mapper app_test chain_test cli_test" set -o pipefail programs/build_helpers/buildstep prepare.sonar 20 "find libraries/[acdenptuw]*/CMakeFiles/*.dir programs/[cdgjsw]*/CMakeFiles/*.dir -type d -print | while read d; do gcov -o \"\$d\" \"\${d/CMakeFiles*.dir//}\"/*.cpp; done >/dev/null; programs/build_helpers/set_sonar_branch sonar-project.properties" du -hs sonar_cache From f91fc829949d7c44f36c392fff69eafe2a83f243 Mon Sep 17 00:00:00 2001 From: abitmore Date: Thu, 2 Jul 2020 18:37:23 -0400 Subject: [PATCH 2/5] Run test cases to generate coverage data --- programs/build_helpers/scan_with_sonar_step_3 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/programs/build_helpers/scan_with_sonar_step_3 b/programs/build_helpers/scan_with_sonar_step_3 index d5105e17e0..35f78082dd 100755 --- a/programs/build_helpers/scan_with_sonar_step_3 +++ b/programs/build_helpers/scan_with_sonar_step_3 @@ -7,6 +7,8 @@ programs/build_helpers/buildstep Prepare 1 "sed -i '/tests/d' libraries/fc/CMake programs/build_helpers/buildstep cmake 5 "cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS=--coverage -DCMAKE_CXX_FLAGS=--coverage -DBoost_USE_STATIC_LIBS=OFF -DCMAKE_CXX_OUTPUT_EXTENSION_REPLACE=ON ." programs/build_helpers/buildstep make.programs 1800 "programs/build_helpers/make_with_sonar bw-output -j 2 witness_node cli_wallet js_operation_serializer get_dev_key network_mapper app_test chain_test cli_test" set -o pipefail +programs/build_helpers/buildstep run.chain_test 600 "libraries/fc/tests/run-parallel-tests.sh tests/chain_test" +programs/build_helpers/buildstep run.cli_test 180 "libraries/fc/tests/run-parallel-tests.sh tests/cli_test" programs/build_helpers/buildstep prepare.sonar 20 "find libraries/[acdenptuw]*/CMakeFiles/*.dir programs/[cdgjsw]*/CMakeFiles/*.dir -type d -print | while read d; do gcov -o \"\$d\" \"\${d/CMakeFiles*.dir//}\"/*.cpp; done >/dev/null; programs/build_helpers/set_sonar_branch sonar-project.properties" du -hs sonar_cache # The first pass is skipped here From 104518a837e40611bb920bef9f82ee9782a51d78 Mon Sep 17 00:00:00 2001 From: abitmore Date: Thu, 2 Jul 2020 21:29:55 -0400 Subject: [PATCH 3/5] Run app_test for code coverage data --- programs/build_helpers/scan_with_sonar_step_3 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/programs/build_helpers/scan_with_sonar_step_3 b/programs/build_helpers/scan_with_sonar_step_3 index 35f78082dd..6d1722a3e5 100755 --- a/programs/build_helpers/scan_with_sonar_step_3 +++ b/programs/build_helpers/scan_with_sonar_step_3 @@ -9,6 +9,7 @@ programs/build_helpers/buildstep make.programs 1800 "programs/build_helpers/make set -o pipefail programs/build_helpers/buildstep run.chain_test 600 "libraries/fc/tests/run-parallel-tests.sh tests/chain_test" programs/build_helpers/buildstep run.cli_test 180 "libraries/fc/tests/run-parallel-tests.sh tests/cli_test" +programs/build_helpers/buildstep run.app_test 180 "tests/app_test" programs/build_helpers/buildstep prepare.sonar 20 "find libraries/[acdenptuw]*/CMakeFiles/*.dir programs/[cdgjsw]*/CMakeFiles/*.dir -type d -print | while read d; do gcov -o \"\$d\" \"\${d/CMakeFiles*.dir//}\"/*.cpp; done >/dev/null; programs/build_helpers/set_sonar_branch sonar-project.properties" du -hs sonar_cache # The first pass is skipped here @@ -17,7 +18,7 @@ programs/build_helpers/buildstep prepare.sonar.part2 1 "cp sonar-project.propert programs/build_helpers/buildstep run.sonar.part2 1500 "which sonar-scanner && sonar-scanner" programs/build_helpers/buildstep prepare.sonar.full 1 "cp sonar-project.properties.bak sonar-project.properties" du -hs sonar_cache -# The second pass, scan all files +# The third pass, scan all files programs/build_helpers/buildstep run.sonar.full 1500 "which sonar-scanner && sonar-scanner" du -hs sonar_cache programs/build_helpers/buildstep end 0 From 4e870a40712e40a330e2a0404d51df1ba9b6107c Mon Sep 17 00:00:00 2001 From: abitmore Date: Fri, 3 Jul 2020 19:20:47 -0400 Subject: [PATCH 4/5] Try to fix sonar-scanner shallow clone warning --- programs/build_helpers/set_sonar_branch | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/programs/build_helpers/set_sonar_branch b/programs/build_helpers/set_sonar_branch index cd2debd034..53656e7307 100755 --- a/programs/build_helpers/set_sonar_branch +++ b/programs/build_helpers/set_sonar_branch @@ -26,20 +26,18 @@ ORIGINAL_TARGET="$( grep 'sonar\.branch\.target' "$1" | sed 's=^.*[:=] *==' )" BRANCH= TARGET="$ORIGINAL_TARGET" -FETCH= -if [ -n "$TRAVIS_PULL_REQUEST" -a "$TRAVIS_PULL_REQUEST" != false ]; then +if [ -n "$TRAVIS_PULL_REQUEST" -a "$TRAVIS_PULL_REQUEST" != "false" ]; then # PRs work per default, remove sonar.branch.* since they only work with sonar.pullrequest.* echo "Detected PR '$TRAVIS_PULL_REQUEST'" TARGET= - FETCH="$TRAVIS_BRANCH" elif [ -n "$TRAVIS_TAG" ]; then # Tag build is either master or testnet echo "Detected tag '$TRAVIS_TAG'" BRANCH="$TRAVIS_BRANCH" case "$TRAVIS_TAG" in - *test*) TARGET=testnet; FETCH=testnet; ;; - *) TARGET=master; FETCH=master; ;; + *test*) TARGET=testnet; ;; + *) TARGET=master; ;; esac else BRANCH="$TRAVIS_BRANCH" @@ -48,29 +46,29 @@ else # Long-lived branches stand for themselves, no target echo "Detected long-lived branch '$TRAVIS_BRANCH'" TARGET= - FETCH="$TRAVIS_BRANCH" ;; *test*release*) # Testnet release branch will be merged into testnet echo "Detected testnet release branch '$TRAVIS_BRANCH'" TARGET=testnet - FETCH=testnet ;; *release*) # Release branch will be merged into default (master) echo "Detected release branch '$TRAVIS_BRANCH'" TARGET=master - FETCH=master ;; *) # All other branches should have sonar.branch.target in their # sonar.properties, leave it unchanged echo "Detected normal branch '$TRAVIS_BRANCH'" - FETCH="$TARGET" esac fi -echo "Branch '$BRANCH', target '$TARGET', fetch target '$FETCH'" +echo "Branch '$BRANCH', target '$TARGET'" + +if [ "$TRAVIS_PULL_REQUEST" = "false" -a "$TRAVIS_REPO_SLUG" = "bitshares/bitshares-core" ]; then + git fetch --unshallow +fi if [ "x$TARGET" != "x$ORIGINAL_TARGET" ]; then clear_branch "$1" @@ -81,11 +79,5 @@ fi if [ -n "$BRANCH" ]; then echo "sonar.branch.name=$BRANCH" >>"$1" fi -#if [ -n "$FETCH" ]; then - # Unfortunately this leads to sonar failing. Apparently it needs a full - # clone, not a shallow one. Since our repo is somewhat large and this is - # only required for blame annotations, disabled for now. - #git fetch --depth=50 origin "$FETCH:$FETCH" -#fi exit 0 From a7cf23f6f4539af9865abcb8104c602854c71076 Mon Sep 17 00:00:00 2001 From: abitmore Date: Fri, 3 Jul 2020 21:53:39 -0400 Subject: [PATCH 5/5] Fix sonar-scanner warning "Could not find ref" --- programs/build_helpers/set_sonar_branch | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/programs/build_helpers/set_sonar_branch b/programs/build_helpers/set_sonar_branch index 53656e7307..fe2f9a0982 100755 --- a/programs/build_helpers/set_sonar_branch +++ b/programs/build_helpers/set_sonar_branch @@ -66,8 +66,10 @@ fi echo "Branch '$BRANCH', target '$TARGET'" -if [ "$TRAVIS_PULL_REQUEST" = "false" -a "$TRAVIS_REPO_SLUG" = "bitshares/bitshares-core" ]; then - git fetch --unshallow +git fetch --no-tags --unshallow +git fetch --no-tags origin +refs/heads/$TRAVIS_BRANCH:refs/remotes/origin/$TRAVIS_BRANCH +if [ -n "$TARGET" ]; then + git fetch --no-tags origin +refs/heads/$TARGET:refs/remotes/origin/$TARGET fi if [ "x$TARGET" != "x$ORIGINAL_TARGET" ]; then