Test fix ups (#45)

* Enable coverage tests
* Fix repo name in check_matrix.py
* Use vars for results paths
* Disable smoketests from collections

tests/integration/targets/ec2_metadata_facts/aliases

@@ -1,3 +1,2 @@
 cloud/aws
 shippable/aws/group2
-shippable/aws/smoketest

tests/utils/shippable/aws.sh

@@ -13,22 +13,7 @@ target="shippable/${cloud}/group${group}/"
 
 stage="${S:-prod}"
 
-changed_all_target="shippable/${cloud}/smoketest/"
-
-if ! ansible-test integration "${changed_all_target}" --list-targets > /dev/null 2>&1; then
-    # no smoketest tests are available for this cloud
-    changed_all_target="none"
-fi
-
-if [ "${group}" == "1" ]; then
-    # only run smoketest tests for group1
-    changed_all_mode="include"
-else
-    # smoketest tests already covered by group1
-    changed_all_mode="exclude"
-fi
-
 # shellcheck disable=SC2086
 ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
     --remote-terminate always --remote-stage "${stage}" \
-    --docker --python "${python}" --changed-all-target "${changed_all_target}" --changed-all-mode "${changed_all_mode}"
+    --docker --python "${python}"

tests/utils/shippable/check_matrix.py

@@ -26,7 +26,7 @@
 def main():  # type: () -> None
     """Main entry point."""
     repo_full_name = os.environ['REPO_FULL_NAME']
-    required_repo_full_name = 'ansible-collections/amazon-core'
+    required_repo_full_name = 'ansible-collections/amazon.aws'
 
     if repo_full_name != required_repo_full_name:
         sys.stderr.write('Skipping matrix check on repo "%s" which is not "%s".\n' % (repo_full_name, required_repo_full_name))

tests/utils/shippable/shippable.sh

@@ -34,20 +34,20 @@ export PATH="${PWD}/bin:${PATH}"
 export PYTHONIOENCODING='utf-8'
 
 if [ "${JOB_TRIGGERED_BY_NAME:-}" == "nightly-trigger" ]; then
-#    COVERAGE=yes
+    COVERAGE=yes
     COMPLETE=yes
 fi
 
-#if [ -n "${COVERAGE:-}" ]; then
-#    # on-demand coverage reporting triggered by setting the COVERAGE environment variable to a non-empty value
-#    export COVERAGE="--coverage"
-#elif [[ "${COMMIT_MESSAGE}" =~ ci_coverage ]]; then
-#    # on-demand coverage reporting triggered by having 'ci_coverage' in the latest commit message
-#    export COVERAGE="--coverage"
-#else
-#    # on-demand coverage reporting disabled (default behavior, always-on coverage reporting remains enabled)
-#    export COVERAGE="--coverage-check"
-#fi
+if [ -n "${COVERAGE:-}" ]; then
+    # on-demand coverage reporting triggered by setting the COVERAGE environment variable to a non-empty value
+    export COVERAGE="--coverage"
+elif [[ "${COMMIT_MESSAGE}" =~ ci_coverage ]]; then
+    # on-demand coverage reporting triggered by having 'ci_coverage' in the latest commit message
+    export COVERAGE="--coverage"
+else
+    # on-demand coverage reporting disabled (default behavior, always-on coverage reporting remains enabled)
+    export COVERAGE="--coverage-check"
+fi
 
 if [ -n "${COMPLETE:-}" ]; then
     # disable change detection triggered by setting the COMPLETE environment variable to a non-empty value
@@ -92,6 +92,7 @@ ansible-galaxy collection install ansible.netcommon
 cd "${cwd}"
 
 export ANSIBLE_COLLECTIONS_PATHS="${HOME}/.ansible/"
+SHIPPABLE_RESULT_DIR="$(pwd)/shippable"
 TEST_DIR="${HOME}/.ansible/collections/ansible_collections/amazon/aws/"
 mkdir -p "${TEST_DIR}"
 cp -aT "${SHIPPABLE_BUILD_DIR}" "${TEST_DIR}"
@@ -109,21 +110,48 @@ function cleanup
             fi
 
             # shellcheck disable=SC2086
-            ansible-test coverage xml --color -v --requirements --group-by command --group-by version ${stub:+"$stub"}
-            cp -a tests/output/reports/coverage=*.xml shippable/codecoverage/
+            ansible-test coverage xml --color --requirements --group-by command --group-by version ${stub:+"$stub"}
+            cp -a tests/output/reports/coverage=*.xml "$SHIPPABLE_RESULT_DIR/codecoverage/"
+
+            # analyze and capture code coverage aggregated by integration test target
+            ansible-test coverage analyze targets generate -v "$SHIPPABLE_RESULT_DIR/testresults/coverage-analyze-targets.json"
+
+            # upload coverage report to codecov.io only when using complete on-demand coverage
+            if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ]; then
+                for file in tests/output/reports/coverage=*.xml; do
+                    flags="${file##*/coverage=}"
+                    flags="${flags%-powershell.xml}"
+                    flags="${flags%.xml}"
+                    # remove numbered component from stub files when converting to tags
+                    flags="${flags//stub-[0-9]*/stub}"
+                    flags="${flags//=/,}"
+                    flags="${flags//[^a-zA-Z0-9_,]/_}"
+
+                    bash <(curl -s https://codecov.io/bash) \
+                        -f "${file}" \
+                        -F "${flags}" \
+                        -n "${test}" \
+                        -t bc371da7-e5d2-4743-93b5-309f81d457a4
+                        -X coveragepy \
+                        -X gcov \
+                        -X fix \
+                        -X search \
+                        -X xcode \
+                    || echo "Failed to upload code coverage report to codecov.io: ${file}"
+                done
+            fi
         fi
     fi
-
     if [ -d  tests/output/junit/ ]; then
-      cp -aT tests/output/junit/ shippable/testresults/
+      cp -aT tests/output/junit/ "$SHIPPABLE_RESULT_DIR/testresults/"
     fi
 
     if [ -d tests/output/data/ ]; then
-      cp -a tests/output/data/ shippable/testresults/
+      cp -a tests/output/data/ "$SHIPPABLE_RESULT_DIR/testresults/"
     fi
 
     if [ -d  tests/output/bot/ ]; then
-      cp -aT tests/output/bot/ shippable/testresults/
+      cp -aT tests/output/bot/ "$SHIPPABLE_RESULT_DIR/testresults/"
     fi
 }