diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index f7a9df22db8c..82e3c786ee64 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml @@ -4,7 +4,7 @@ description: Runs on-device tests. runs: using: "composite" steps: - - name: Install requirements + - name: Install Requirements run: | pip3 install grpcio==1.38.0 grpcio-tools==1.38.0 shell: bash @@ -12,15 +12,14 @@ runs: run: | python -m grpc_tools.protoc -Itools/ --python_out=tools/ --grpc_python_out=tools/ tools/on_device_tests_gateway.proto shell: bash - - name: Set up Cloud SDK + - name: Set Up Cloud SDK uses: isarkis/setup-gcloud@40dce7857b354839efac498d3632050f568090b6 # v1.1.1 - name: Set env vars - env: - WORKFLOW: ${{ github.workflow }} run: | echo "PROJECT_NAME=$(gcloud config get-value project)" >> $GITHUB_ENV - echo "GITHUB_RUN_NUMBER=${GITHUB_RUN_NUMBER}" >> $GITHUB_ENV - echo "WORKFLOW=${WORKFLOW}" >> $GITHUB_ENV + + # Test results and logs + echo "GCS_RESULTS_PATH=gs://cobalt-unittest-storage/results/${{ matrix.name }}/${{ github.run_id }}_${{ matrix.shard }}" >> $GITHUB_ENV # Boot loader env if [ "${COBALT_EVERGREEN_LOADER}" != "null" ]; then @@ -43,8 +42,9 @@ runs: echo "USE_SHARDING=1" >> $GITHUB_ENV fi shell: bash - - name: run ${{ env.SHARD_NAME }} tests on ${{ matrix.platform }} platform + - name: Run ${{ env.SHARD_NAME }} Tests on ${{ matrix.platform }} Platform env: + GCS_PATH: gs://${{ env.PROJECT_NAME }}-test-artifacts/${{ github.workflow }}/${{ github.run_number }}/${{ matrix.platform }}_${{ matrix.config }} GITHUB_SHA: ${{ github.sha }} GITHUB_TOKEN: ${{ github.token }} GITHUB_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} @@ -58,14 +58,16 @@ runs: GITHUB_COMMIT_AUTHOR_USERNAME: ${{ github.event.commits[0].author.username }} GITHUB_COMMIT_AUTHOR_EMAIL: ${{ github.event.commits[0].author.email }} GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GITHUB_WORKFLOW: ${{ github.workflow }} run: | set -uxe python3 -u tools/on_device_tests_gateway_client.py \ --token ${GITHUB_TOKEN} \ --change_id ${GITHUB_PR_NUMBER:-postsubmit} \ trigger \ - --test_type ${{ env.TEST_TYPE }} \ + --test_type ${TEST_TYPE} \ --platform ${{ matrix.target_platform }} \ --config ${{ matrix.config }} \ --tag cobalt_github_${GITHUB_EVENT_NAME} \ @@ -77,10 +79,11 @@ runs: ${DIMENSION:+"--dimension" "$DIMENSION"} \ ${USE_SHARDING:+"--unittest_shard_index" "${{ matrix.shard }}"} \ ${ON_DEVICE_TEST_ATTEMPTS:+"--test_attempts" "$ON_DEVICE_TEST_ATTEMPTS"} \ - --archive_path gs://${PROJECT_NAME}-test-artifacts/${WORKFLOW}/${GITHUB_RUN_NUMBER}/${{ matrix.platform }}_${{ matrix.config }}/artifacts.tar \ + --archive_path "${GCS_PATH}/artifacts.tar" \ + --gcs_result_path "${GCS_RESULTS_PATH}" \ --label github \ --label ${GITHUB_EVENT_NAME} \ - --label ${WORKFLOW} \ + --label ${GITHUB_WORKFLOW} \ --label actor-${GITHUB_ACTOR} \ --label actor_id-${GITHUB_ACTOR_ID} \ --label triggering_actor-${GITHUB_TRIGGERING_ACTOR} \ @@ -89,3 +92,73 @@ runs: --label author-${GITHUB_PR_HEAD_USER_LOGIN:-$GITHUB_COMMIT_AUTHOR_USERNAME} \ --label author_id-${GITHUB_PR_HEAD_USER_ID:-$GITHUB_COMMIT_AUTHOR_EMAIL} shell: bash + - name: Download ${{ matrix.platform }} Test Results + if: always() && env.TEST_TYPE == 'unit_test' + run: | + # Don't break on error (-e), some commands are expected to fail. + set -ux + + COBALT_LOGS_DIR="${GITHUB_WORKSPACE}/cobalt_logs" + UNIT_TEST_RESULT_PATH="${GITHUB_WORKSPACE}/unit-test-results" + COBALT_XMLS_FILENAME="cobalt_xmls.zip" + + # Forward environment variables for uploading artifacts in later steps. + echo "UNIT_TEST_RESULT_PATH=${UNIT_TEST_RESULT_PATH}" >> $GITHUB_ENV + echo "COBALT_LOGS_DIR=${COBALT_LOGS_DIR}" >> $GITHUB_ENV + + mkdir -p "${GITHUB_WORKSPACE}/test_results" + cd "${GITHUB_WORKSPACE}/test_results" + + i=0 + # Try downloading the results for 6x 10 seconds before giving up. + while [ $i -lt 6 ]; do + # The results are uploaded after the test has completed. + sleep 10 + + # The log files are named by the device lab test driver. + COBALT_ERROR_LOG="webDriverTestLog.ERROR" + COBALT_INFO_LOG="webDriverTestLog.INFO" + + # This command will fail until the results have been uploaded. + gsutil cp "${GCS_RESULTS_PATH}/${COBALT_ERROR_LOG}" . + gsutil cp "${GCS_RESULTS_PATH}/${COBALT_INFO_LOG}" . + gsutil cp "${GCS_RESULTS_PATH}/${COBALT_XMLS_FILENAME}" . + + # Break if all files were downloaded. + if [[ -f "${COBALT_XMLS_FILENAME}" && -f "${COBALT_ERROR_LOG}" && -f "${COBALT_INFO_LOG}" ]]; then + break + fi + + i=$(( ${i} + 1 )) + done + + # Print device logs. Might fail if the log file failed to download. + cat ${COBALT_ERROR_LOG} + + # Rename log files for archiving to not expose legacy weirdness. + mkdir -p "${COBALT_LOGS_DIR}/${{ matrix.platform }}/" + mv "${COBALT_ERROR_LOG}" "${COBALT_LOGS_DIR}/${{ matrix.platform }}/stderr_${{ matrix.shard }}.log" + mv "${COBALT_INFO_LOG}" "${COBALT_LOGS_DIR}/${{ matrix.platform }}/stdout_${{ matrix.shard }}.log" + + # Prepare unit test results for DataDog upload. + RESULT_PATH=${UNIT_TEST_RESULT_PATH}/${{ matrix.platform }}/${{ matrix.shard }}/ + mkdir -p ${RESULT_PATH} + + # Set tags for test differentiation. + tags="platform:${{ matrix.platform }}" + echo $tags > ${UNIT_TEST_RESULT_PATH}/${{ matrix.platform }}/TAGS + + unzip ${COBALT_XMLS_FILENAME} -d ${RESULT_PATH} + shell: bash + - name: Archive Unit Test Logs + uses: actions/upload-artifact@v3 + if: always() && env.TEST_TYPE == 'unit_test' + with: + name: Device logs + path: ${{ env.COBALT_LOGS_DIR }}/ + - name: Archive Unit Test Results + uses: actions/upload-artifact@v3 + if: always() && env.TEST_TYPE == 'unit_test' + with: + name: unit-test-results + path: ${{ env.UNIT_TEST_RESULT_PATH }}/ diff --git a/.github/actions/on_host_test/action.yaml b/.github/actions/on_host_test/action.yaml index f4fac30f3365..2276acdc37ed 100644 --- a/.github/actions/on_host_test/action.yaml +++ b/.github/actions/on_host_test/action.yaml @@ -96,7 +96,7 @@ runs: fi fi - name: Populate TAGS for unit test report - if: (success() || failure()) && steps.run-tests.outputs.test_type == 'unit_tests' + if: always() && steps.run-tests.outputs.test_type == 'unit_tests' shell: bash run: | # Set tags for test differentiation. @@ -105,7 +105,7 @@ runs: echo $tags > ${TEST_RESULTS_DIR}/${{ matrix.platform }}/TAGS - name: Archive unit test results uses: actions/upload-artifact@v3 - if: (success() || failure()) && steps.run-tests.outputs.test_type == 'unit_tests' + if: always() && steps.run-tests.outputs.test_type == 'unit_tests' with: name: unit-test-results path: ${{env.TEST_RESULTS_DIR}}/ diff --git a/tools/on_device_tests_gateway.proto b/tools/on_device_tests_gateway.proto index 58e6558faef5..61ef36c05740 100644 --- a/tools/on_device_tests_gateway.proto +++ b/tools/on_device_tests_gateway.proto @@ -52,6 +52,7 @@ message OnDeviceTestsCommand { string start_timeout = 20; string test_timeout = 21; string builder_url = 22; + string gcs_result_path = 23; } // Working directory and command line arguments to be passed to the gateway. diff --git a/tools/on_device_tests_gateway_client.py b/tools/on_device_tests_gateway_client.py index 3ed5637826d2..8d60ee3f3dc6 100644 --- a/tools/on_device_tests_gateway_client.py +++ b/tools/on_device_tests_gateway_client.py @@ -77,6 +77,7 @@ def run_trigger_command(self, workdir: str, args: argparse.Namespace): config=args.config, tag=args.tag, labels=args.label, + gcs_result_path=args.gcs_result_path, builder_name=args.builder_name, builder_url=args.builder_url, change_id=args.change_id, @@ -192,6 +193,10 @@ def main(): 'if any. Saved with performance test results') trigger_parser.add_argument( '--builder_url', type=str, help='Url to the run, if any.') + trigger_parser.add_argument( + '--gcs_result_path', + type=str, + help='GCS url where test result files should be uploaded.') trigger_parser.add_argument( '-n', '--build_number',