From 86d8e349b4e810c0dca56d5202f893628969970a Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 3 Aug 2020 20:01:00 +0200 Subject: [PATCH] Status of quarantined tests is stored in Github Issue (#10119) --- .github/workflows/ci.yml | 44 +--- .github/workflows/quarantined.yaml | 116 +++++++++ CI.rst | 63 ++++- scripts/ci/docker-compose/base.yml | 4 + scripts/ci/in_container/entrypoint_ci.sh | 32 ++- .../in_container/quarantine_issue_header.md | 32 +++ scripts/ci/in_container/run_ci_tests.sh | 25 ++ .../update_quarantined_test_status.py | 243 ++++++++++++++++++ scripts/ci/libraries/_initialization.sh | 8 + scripts/ci/testing/ci_run_airflow_testing.sh | 18 ++ setup.py | 3 +- 11 files changed, 539 insertions(+), 49 deletions(-) create mode 100644 .github/workflows/quarantined.yaml create mode 100644 scripts/ci/in_container/quarantine_issue_header.md create mode 100755 scripts/ci/in_container/update_quarantined_test_status.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab6c745719b2..3735d74626c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,7 @@ jobs: steps: - uses: potiuk/cancel-workflow-runs@v1 with: + workflow: ci.yaml token: ${{ secrets.GITHUB_TOKEN }} static-checks: @@ -62,7 +63,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '3.x' + python-version: '3.7' - name: Cache pre-commit env uses: actions/cache@v2 env: @@ -100,7 +101,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '3.x' + python-version: '3.7' - name: "Free space" run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Build CI image ${{ matrix.python-version }}" @@ -154,7 +155,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '3.x' + python-version: '3.7' - name: "Free space" run: ./scripts/ci/tools/ci_free_space_on_ci.sh - uses: engineerd/setup-kind@v0.4.0 @@ -204,7 +205,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '3.x' + python-version: '3.7' - name: "Free space" run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Build CI image ${{ matrix.python-version }}" @@ -234,7 +235,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '3.x' + python-version: '3.7' - name: "Free space" run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Build CI image ${{ matrix.python-version }}" @@ -262,38 +263,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '3.x' - - name: "Free space" - run: ./scripts/ci/tools/ci_free_space_on_ci.sh - - name: "Build CI image ${{ matrix.python-version }}" - run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - - name: "Tests" - run: ./scripts/ci/testing/ci_run_airflow_testing.sh - - tests-quarantined: - timeout-minutes: 80 - name: "${{matrix.test-type}}:Pg${{matrix.postgres-version}},Py${{matrix.python-version}}" - runs-on: ubuntu-latest - continue-on-error: true - needs: [trigger-tests] - strategy: - matrix: - python-version: [3.6] - postgres-version: [9.6] - test-type: [Quarantined] - fail-fast: false - env: - BACKEND: postgres - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - POSTGRES_VERSION: ${{ matrix.postgres-version }} - RUN_TESTS: "true" - TEST_TYPE: ${{ matrix.test-type }} - if: needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request' - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '3.x' + python-version: '3.7' - name: "Free space" run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Build CI image ${{ matrix.python-version }}" diff --git a/.github/workflows/quarantined.yaml b/.github/workflows/quarantined.yaml new file mode 100644 index 000000000000..75135ec8dc93 --- /dev/null +++ b/.github/workflows/quarantined.yaml @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: Quarantined Build +on: + schedule: + # Run quarantined builds 4 times a day to gather better quarantine stats + - cron: '35 */6 * * *' + push: + branches: ['master', 'v1-10-test', 'v1-10-stable'] + pull_request: + branches: ['master', 'v1-10-test', 'v1-10-stable'] + +env: + MOUNT_LOCAL_SOURCES: "true" + FORCE_ANSWER_TO_QUESTIONS: "yes" + SKIP_CHECK_REMOTE_IMAGE: "true" + SKIP_CI_IMAGE_CHECK: "true" + DB_RESET: "true" + VERBOSE: "true" + UPGRADE_TO_LATEST_CONSTRAINTS: ${{ github.event_name == 'push' || github.event_name == 'scheduled' }} + PYTHON_MAJOR_MINOR_VERSION: 3.6 + USE_GITHUB_REGISTRY: "true" + CACHE_IMAGE_PREFIX: ${{ github.repository }} + CACHE_REGISTRY_USERNAME: ${{ github.actor }} + CACHE_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + +jobs: + + cancel-previous-workflow-run: + timeout-minutes: 60 + name: "Cancel previous workflow run" + runs-on: ubuntu-latest + steps: + - uses: potiuk/cancel-workflow-runs@v1 + with: + workflow: quarantined.yml + token: ${{ secrets.GITHUB_TOKEN }} + + trigger-tests: + timeout-minutes: 5 + name: "Checks if tests should be run" + runs-on: ubuntu-latest + needs: [cancel-previous-workflow-run] + outputs: + run-tests: ${{ steps.trigger-tests.outputs.run-tests }} + steps: + - uses: actions/checkout@v2 + - name: "Check if tests should be run" + run: "./scripts/ci/tools/ci_check_if_tests_should_be_run.sh" + id: trigger-tests + + tests-quarantined: + timeout-minutes: 80 + name: "Quarantined tests" + runs-on: ubuntu-latest + continue-on-error: true + needs: [trigger-tests] + strategy: + matrix: + python-version: [3.6] + postgres-version: [9.6] + fail-fast: false + env: + BACKEND: postgres + PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} + POSTGRES_VERSION: ${{ matrix.postgres-version }} + RUN_TESTS: "true" + TEST_TYPE: Quarantined + NUM_RUNS: 10 + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + if: needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request' + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: "Set issue id for master" + if: github.ref == 'refs/heads/master' + run: | + echo "::set-env name=ISSUE_ID::86" + - name: "Set issue id for v1-10-stable" + if: github.ref == 'refs/heads/v1-10-stable' + run: | + echo "::set-env name=ISSUE_ID::10127" + - name: "Set issue id for v1-10-test" + if: github.ref == 'refs/heads/v1-10-test' + run: | + echo "::set-env name=ISSUE_ID::10128" + - name: "Free space" + run: ./scripts/ci/tools/ci_free_space_on_ci.sh + - name: "Build CI image ${{ matrix.python-version }}" + run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh + - name: "Tests" + run: ./scripts/ci/testing/ci_run_airflow_testing.sh + - uses: actions/upload-artifact@v2 + name: Upload Quarantine test results + if: always() + with: + name: 'quarantined_tests' + path: 'files/test_result.xml' diff --git a/CI.rst b/CI.rst index 41d22fb62011..2844a5c3c20c 100644 --- a/CI.rst +++ b/CI.rst @@ -67,8 +67,8 @@ The following components are part of the CI infrastructure CI run types ============ -The following CI Job runs are currently run for Apache Airflow, and each of the runs have different -purpose and context. +The following CI Job run types are currently run for Apache Airflow (run by ci.yaml workflow and +quarantined.yaml workflows) and each of the run types have different purpose and context. Pull request run ---------------- @@ -126,7 +126,17 @@ DockerHub when pushing ``v1-10-stable`` manually. All runs consist of the same jobs, but the jobs behave slightly differently or they are skipped in different run categories. Here is a summary of the run categories with regards of the jobs they are running. Those jobs often have matrix run strategy which runs several different variations of the jobs -(with different Backend type / Python version, type of the tests to run for example) +(with different Backend type / Python version, type of the tests to run for example). The following chapter +describes the workflows that execute for each run. + +Workflows +========= + +CI Build Workflow +----------------- + +This workflow is a regular workflow that performs the regular checks - none of the jobs should fail. +The tests to run do not contain quarantined tests. +---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+ | Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | @@ -148,8 +158,6 @@ Those jobs often have matrix run strategy which runs several different variation +---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+ | Tests Kubernetes | Run Kubernetes test | Yes (if tests-triggered) | Yes | Yes * | +---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+ -| Quarantined tests | Those are tests that are flaky and we need to fix them | Yes (if tests-triggered) | Yes | Yes * | -+---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+ | Test OpenAPI client gen | Tests if OpenAPIClient continues to generate | Yes | Yes | Yes * | +---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+ | Helm tests | Runs tests for the Helm chart | Yes | Yes | Yes * | @@ -164,3 +172,48 @@ Those jobs often have matrix run strategy which runs several different variation +---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+ | Tag Repo nightly | Tags the repository with nightly tagIt is a lightweight tag that moves nightly | - | - | Yes. Triggers DockerHub build for public registry | +---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+ + +Quarantined build workflow +-------------------------- + +This workflow runs only quarantined tests. Those tests do not fail the build even if some tests fail (only if +the whole pytest execution fails). Instead this workflow updates one of the issues where we keep status +of quarantined tests. Once the test succeeds in NUM_RUNS subsequent runs, it is marked as stable and +can be removed from quarantine. You can read more about quarantine in ``_ + +The issues are only updated if the test is run as direct push or scheduled run and only in the +``apache/airflow`` repository - so that the issues are not updated in forks. + +The issues that gets updated are different for different branches: + +* master: `Quarantine tests master `_ +* v1-10-stable: `Quarantine tests v1-10-stable `_ +* v1-10-test: `Quarantine tests v1-10-test `_ + ++---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+ +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | ++===========================+================================================================================================================+====================================+=================================+======================================================================+ +| Cancel previous workflow | Cancels the previously running workflow run if there is one running | Yes | Yes | Yes * | ++---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+ +| Trigger tests | Checks if tests should be triggered | Yes | Yes | Yes * | ++---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+ +| Quarantined tests | Those are tests that are flaky and we need to fix them | Yes (if tests-triggered) | Yes (Updates quarantine issue) | Yes * (updates quarantine issue) | ++---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+ + +Cancel other workflow runs workflow +----------------------------------- + +This workflow is run only on schedule (every 5 minutes) it's only purpose is to cancel other running +``CI Build`` workflows if important jobs failed in those runs. This is to save runners for other runs +in case we know that the build will not succeed anyway without some basic fixes to static checks or +documentation - effectively implementing missing "fail-fast" (on a job level) in Github Actions +similar to fail-fast in matrix strategy. + +The jobs that are considered as "fail-fast" are: + +* Static checks +* Docs +* Prepare Backport packages +* Helm tests +* Build Prod Image +* TTest OpenAPI client gen diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml index 0feea60ce062..9a364a22ae9c 100644 --- a/scripts/ci/docker-compose/base.yml +++ b/scripts/ci/docker-compose/base.yml @@ -39,6 +39,10 @@ services: - RUN_INTEGRATION_TESTS - ONLY_RUN_LONG_RUNNING_TESTS - ONLY_RUN_QUARANTINED_TESTS + - GITHUB_TOKEN + - GITHUB_REPOSITORY + - ISSUE_ID + - NUM_RUNS - BREEZE - INSTALL_AIRFLOW_VERSION - DB_RESET diff --git a/scripts/ci/in_container/entrypoint_ci.sh b/scripts/ci/in_container/entrypoint_ci.sh index d006eaa6652b..fa6091c37125 100755 --- a/scripts/ci/in_container/entrypoint_ci.sh +++ b/scripts/ci/in_container/entrypoint_ci.sh @@ -161,11 +161,12 @@ if [[ "${RUN_TESTS}" != "true" ]]; then fi set -u +export RESULT_LOG_FILE="/files/test_result.xml" + if [[ "${CI}" == "true" ]]; then EXTRA_PYTEST_ARGS=( "--verbosity=0" "--strict-markers" - "--instafail" "--durations=100" "--cov=airflow/" "--cov-config=.coveragerc" @@ -174,6 +175,7 @@ if [[ "${CI}" == "true" ]]; then "--maxfail=50" "--pythonwarnings=ignore::DeprecationWarning" "--pythonwarnings=ignore::PendingDeprecationWarning" + "--junitxml=${RESULT_LOG_FILE}" ) else EXTRA_PYTEST_ARGS=() @@ -187,25 +189,43 @@ if [[ ${#@} -gt 0 && -n "$1" ]]; then fi if [[ -n ${RUN_INTEGRATION_TESTS:=""} ]]; then + # Integration tests for INT in ${RUN_INTEGRATION_TESTS} do EXTRA_PYTEST_ARGS+=("--integration" "${INT}") done - EXTRA_PYTEST_ARGS+=("-rpfExX") + EXTRA_PYTEST_ARGS+=( + # timeouts in seconds for individual tests + "--setup-timeout=20" + "--execution-timeout=60" + "--teardown-timeout=20" + ) + elif [[ ${ONLY_RUN_LONG_RUNNING_TESTS:=""} == "true" ]]; then EXTRA_PYTEST_ARGS+=( "-m" "long_running" "--include-long-running" "--verbosity=1" - "--reruns" "3" - "--timeout" "90") + "--setup-timeout=30" + "--execution-timeout=120" + "--teardown-timeout=30" + ) elif [[ ${ONLY_RUN_QUARANTINED_TESTS:=""} == "true" ]]; then EXTRA_PYTEST_ARGS+=( "-m" "quarantined" "--include-quarantined" "--verbosity=1" - "--reruns" "3" - "--timeout" "90") + "--setup-timeout=10" + "--execution-timeout=50" + "--teardown-timeout=10" + ) +else + # Core tests + EXTRA_PYTEST_ARGS+=( + "--setup-timeout=10" + "--execution-timeout=30" + "--teardown-timeout=10" + ) fi ARGS=("${EXTRA_PYTEST_ARGS[@]}" "${TESTS_TO_RUN[@]}") diff --git a/scripts/ci/in_container/quarantine_issue_header.md b/scripts/ci/in_container/quarantine_issue_header.md new file mode 100644 index 000000000000..d672a4de7bf4 --- /dev/null +++ b/scripts/ci/in_container/quarantine_issue_header.md @@ -0,0 +1,32 @@ + + +# Quarantined issues + +Please do not update status or list of the issues manually. It is automatically updated during +Quarantine workflow, when the workflow executes in the context of Apache Airflow repository. +This happens on schedule (4 times a day) or when a change has been merged or pushed +to the relevant branch. + +You can update "Comment" column in the issue list - the update process will read and preserve this column. + +# Status update +Last status update (UTC): {{ DATE_UTC_NOW }} + +# List of Quarantined issues diff --git a/scripts/ci/in_container/run_ci_tests.sh b/scripts/ci/in_container/run_ci_tests.sh index dc86cf0bc0c4..6d3648026d1b 100755 --- a/scripts/ci/in_container/run_ci_tests.sh +++ b/scripts/ci/in_container/run_ci_tests.sh @@ -33,6 +33,31 @@ if [[ "${RES}" == "0" && ${CI:="false"} == "true" ]]; then bash <(curl -s https://codecov.io/bash) fi +MAIN_GITHUB_REPOSITORY="apache/airflow" + +if [[ ${ONLY_RUN_QUARANTINED_TESTS:=} = "true" ]]; then + if [[ ${GITHUB_REPOSITORY} == "${MAIN_GITHUB_REPOSITORY}" ]]; then + if [[ ${RES} == "1" || ${RES} == "0" ]]; then + echo + echo "Pytest exited with ${RES} result. Updating Quarantine Issue!" + echo + "${IN_CONTAINER_DIR}/update_quarantined_test_status.py" "${RESULT_LOG_FILE}" + else + echo + echo "Pytest exited with ${RES} result. NOT Updating Quarantine Issue!" + echo + fi + else + echo + echo "Github repository '${GITHUB_REPOSITORY}'. NOT Updating Quarantine Issue!" + echo + fi +else + echo + echo "Regular tests. NOT Updating Quarantine Issue!" + echo +fi + if [[ ${CI:=} == "true" ]]; then send_airflow_logs_to_file_io fi diff --git a/scripts/ci/in_container/update_quarantined_test_status.py b/scripts/ci/in_container/update_quarantined_test_status.py new file mode 100755 index 000000000000..179ff91b37b4 --- /dev/null +++ b/scripts/ci/in_container/update_quarantined_test_status.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import re +import sys +from datetime import datetime +from os.path import dirname, join, realpath +from typing import Dict, List, NamedTuple, Optional +from urllib.parse import urlsplit + +import jinja2 +from bs4 import BeautifulSoup +from github3 import login +from jinja2 import StrictUndefined +from tabulate import tabulate + + +class TestResult(NamedTuple): + test_id: str + file: str + name: str + classname: str + line: str + result: bool + + +class TestHistory(NamedTuple): + test_id: str + name: str + url: str + states: List[bool] + comment: str + + +test_results = [] + +user = "" +repo = "" +issue_id = 0 +num_runs = 10 + +url_pattern = re.compile(r'\[([^]]*)]\(([^)]*)\)') + +status_map: Dict[str, bool] = { + ":heavy_check_mark:": True, + ":x:": False, +} + +reverse_status_map: Dict[bool, str] = {status_map[key]: key for key in status_map.keys()} + + +def get_url(result: TestResult) -> str: + return f"[{result.name}](https://github.com/{user}/{repo}/blob/" \ + f"master/{result.file}?test_id={result.test_id}#L{result.line})" + + +def parse_state_history(history_string: str) -> List[bool]: + history_array = history_string.split(' ') + status_array: List[bool] = [] + for value in history_array: + if value: + status_array.append(status_map[value]) + return status_array + + +def parse_test_history(line: str) -> Optional[TestHistory]: + values = line.split("|") + match_url = url_pattern.match(values[1].strip()) + if match_url: + name = match_url.group(1) + url = match_url.group(0) + http_url = match_url.group(2) + parsed_url = urlsplit(http_url) + the_id = parsed_url[3].split("=")[1] + comment = values[4] if len(values) >= 5 else "" + # noinspection PyBroadException + try: + states = parse_state_history(values[3]) + except Exception: + states = [] + return TestHistory( + test_id=the_id, + name=name, + states=states, + url=url, + comment=comment, + ) + return None + + +def parse_body(body: str) -> Dict[str, TestHistory]: + parse = False + test_history_map: Dict[str, TestHistory] = {} + for line in body.splitlines(keepends=False): + if line.startswith("|-"): + parse = True + continue + if parse: + if not line.startswith("|"): + break + # noinspection PyBroadException + try: + status = parse_test_history(line) + except Exception: + continue + if status: + test_history_map[status.test_id] = status + return test_history_map + + +def update_test_history(history: TestHistory, last_status: bool): + print(f"Adding status to test history: {history}, {last_status}") + return TestHistory( + test_id=history.test_id, + name=history.name, + url=history.url, + states=([last_status] + history.states)[0:num_runs], + comment=history.comment, + ) + + +def create_test_history(result: TestResult) -> TestHistory: + print(f"Creating test history {result}") + return TestHistory( + test_id=result.test_id, + name=result.name, + url=get_url(result), + states=[result.result], + comment="" + ) + + +def get_history_status(history: TestHistory): + if len(history.states) < num_runs: + if all(history.states): + return "So far, so good" + return "Flaky" + if all(history.states): + return "Stable" + if all(history.states[0:num_runs - 1]): + return "Just one more" + if all(history.states[0:int(num_runs / 2)]): + return "Almost there" + return "Flaky" + + +def get_table(history_map: Dict[str, TestHistory]) -> str: + headers = ["Test", "Last run", f"Last {num_runs} runs", "Status", "Comment"] + the_table: List[List[str]] = [] + for ordered_key in sorted(history_map.keys()): + history = history_map[ordered_key] + the_table.append([ + history.url, + "Succeeded" if history.states[0] else "Failed", + " ".join([reverse_status_map[state] for state in history.states]), + get_history_status(history), + history.comment + ]) + return tabulate(the_table, headers, tablefmt="github") + + +if __name__ == '__main__': + if len(sys.argv) < 2: + print("Provide XML JUNIT FILE as first argument") + sys.exit(1) + + with open(sys.argv[1], "r") as f: + text = f.read() + y = BeautifulSoup(text, "html.parser") + res = y.testsuites.testsuite.findAll("testcase") + for test in res: + print("Parsing: " + test['classname'] + "::" + test['name']) + if len(test.contents) > 0 and test.contents[0].name == 'skipped': + print(f"skipping {test['name']}") + continue + test_results.append(TestResult( + test_id=test['classname'] + "::" + test['name'], + file=test['file'], + line=test['line'], + name=test['name'], + classname=test['classname'], + result=len(test.contents) == 0 + )) + + token = os.environ.get("GITHUB_TOKEN") + print(f"Token: {token}") + github_repository = os.environ.get('GITHUB_REPOSITORY') + if not github_repository: + raise Exception("Github Repository must be defined!") + user, repo = github_repository.split("/") + print(f"User: {user}, Repo: {repo}") + issue_id = int(os.environ.get('ISSUE_ID', 0)) + num_runs = int(os.environ.get('NUM_RUNS', 10)) + + if issue_id == 0: + raise Exception("You need to define ISSUE_ID as environment variable") + + gh = login(token=token) + + quarantined_issue = gh.issue(user, repo, issue_id) + print("-----") + print(quarantined_issue.body) + print("-----") + parsed_test_map = parse_body(quarantined_issue.body) + new_test_map: Dict[str, TestHistory] = {} + + for test_result in test_results: + previous_results = parsed_test_map.get(test_result.test_id) + if previous_results: + updated_results = update_test_history( + previous_results, test_result.result) + new_test_map[previous_results.test_id] = updated_results + else: + new_history = create_test_history(test_result) + new_test_map[new_history.test_id] = new_history + table = get_table(new_test_map) + print() + print("Result:") + print() + print(table) + print() + with open(join(dirname(realpath(__file__)), "quarantine_issue_header.md"), "r") as f: + header = jinja2.Template(f.read(), autoescape=True, undefined=StrictUndefined).\ + render(DATE_UTC_NOW=datetime.utcnow()) + quarantined_issue.edit(title=None, + body=header + "\n\n" + str(table), + state='open' if len(test_results) > 0 else 'closed') diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index 49afef0f624f..08a6b694c801 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -287,6 +287,14 @@ function get_environment_for_builds_on_ci() { else export CI_EVENT_TYPE="push" fi + elif [[ "${LOCAL_CI_TESTING:=}" == "true" ]]; then + export CI_TARGET_REPO="apache/airflow" + export CI_TARGET_BRANCH="${DEFAULT_BRANCH:="master"}" + export CI_BUILD_ID="0" + export CI_JOB_ID="0" + export CI_EVENT_TYPE="pull_request" + export CI_SOURCE_REPO="apache/airflow" + export CI_SOURCE_BRANCH="${DEFAULT_BRANCH:="master"}" else echo echo "ERROR! Unknown CI environment. Exiting" diff --git a/scripts/ci/testing/ci_run_airflow_testing.sh b/scripts/ci/testing/ci_run_airflow_testing.sh index 884b164f5590..091d87f3fc79 100755 --- a/scripts/ci/testing/ci_run_airflow_testing.sh +++ b/scripts/ci/testing/ci_run_airflow_testing.sh @@ -30,6 +30,7 @@ fi function run_airflow_testing_in_docker() { set +u + set +e # shellcheck disable=SC2016 docker-compose --log-level INFO \ -f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \ @@ -37,7 +38,18 @@ function run_airflow_testing_in_docker() { "${INTEGRATIONS[@]}" \ "${DOCKER_COMPOSE_LOCAL[@]}" \ run airflow "${@}" + EXIT_CODE=$? + if [[ ${ONLY_RUN_QUARANTINED_TESTS:=} == "true" ]]; then + if [[ ${EXIT_CODE} == "1" ]]; then + echo "Some Quarantined tests failed. but we recorded it in an issue" + EXIT_CODE="0" + else + echo "All Quarantined tests succeeded" + fi + fi set -u + set -e + return "${EXIT_CODE}" } get_environment_for_builds_on_ci @@ -93,6 +105,7 @@ elif [[ ${TEST_TYPE:=} == "Long" ]]; then export ONLY_RUN_LONG_RUNNING_TESTS="true" elif [[ ${TEST_TYPE:=} == "Quarantined" ]]; then export ONLY_RUN_QUARANTINED_TESTS="true" + # Do not fail in quarantined tests fi for _INT in ${ENABLED_INTEGRATIONS} @@ -103,4 +116,9 @@ done RUN_INTEGRATION_TESTS=${RUN_INTEGRATION_TESTS:=""} + run_airflow_testing_in_docker "${@}" + +if [[ ${TEST_TYPE:=} == "Quarantined" ]]; then + export ONLY_RUN_QUARANTINED_TESTS="true" +fi diff --git a/setup.py b/setup.py index 60497cffdbf9..68f10f24b7b8 100644 --- a/setup.py +++ b/setup.py @@ -451,6 +451,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version 'flake8-colors', 'flaky', 'freezegun', + 'github3.py', 'gitpython', 'ipdb', 'jira', @@ -466,7 +467,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version 'pytest-cov', 'pytest-instafail', 'pytest-rerunfailures', - 'pytest-timeout', + 'pytest-timeouts', 'pytest-xdist', 'pywinrm', 'qds-sdk>=1.9.6',