Status of quarantined tests is stored in Github Issue

.github/workflows/ci.yml

@@ -49,6 +49,7 @@ jobs:
     steps:
       - uses: potiuk/cancel-workflow-runs@v1
         with:
+          workflow: ci.yaml
           token: ${{ secrets.GITHUB_TOKEN }}
 
   static-checks:
@@ -62,7 +63,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.x'
+          python-version: '3.7'
       - name: Cache pre-commit env
         uses: actions/cache@v2
         env:
@@ -100,7 +101,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.x'
+          python-version: '3.7'
       - name: "Free space"
         run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Build CI image ${{ matrix.python-version }}"
@@ -154,7 +155,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.x'
+          python-version: '3.7'
       - name: "Free space"
         run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - uses: engineerd/setup-kind@v0.4.0
@@ -204,7 +205,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.x'
+          python-version: '3.7'
       - name: "Free space"
         run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Build CI image ${{ matrix.python-version }}"
@@ -234,7 +235,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.x'
+          python-version: '3.7'
       - name: "Free space"
         run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Build CI image ${{ matrix.python-version }}"
@@ -262,38 +263,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.x'
-      - name: "Free space"
-        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
-      - name: "Build CI image ${{ matrix.python-version }}"
-        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
-      - name: "Tests"
-        run: ./scripts/ci/testing/ci_run_airflow_testing.sh
-
-  tests-quarantined:
-    timeout-minutes: 80
-    name: "${{matrix.test-type}}:Pg${{matrix.postgres-version}},Py${{matrix.python-version}}"
-    runs-on: ubuntu-latest
-    continue-on-error: true
-    needs: [trigger-tests]
-    strategy:
-      matrix:
-        python-version: [3.6]
-        postgres-version: [9.6]
-        test-type: [Quarantined]
-      fail-fast: false
-    env:
-      BACKEND: postgres
-      PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
-      POSTGRES_VERSION: ${{ matrix.postgres-version }}
-      RUN_TESTS: "true"
-      TEST_TYPE: ${{ matrix.test-type }}
-    if: needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request'
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.x'
+          python-version: '3.7'
       - name: "Free space"
         run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Build CI image ${{ matrix.python-version }}"

.github/workflows/quarantined.yaml

@@ -0,0 +1,116 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: Quarantined Build
+on:
+  schedule:
+    # Run quarantined builds 4 times a day to gather better quarantine stats
+    - cron: '35 */6 * * *'
+  push:
+    branches: ['master', 'v1-10-test', 'v1-10-stable']
+  pull_request:
+    branches: ['master', 'v1-10-test', 'v1-10-stable']
+
+env:
+  MOUNT_LOCAL_SOURCES: "true"
+  FORCE_ANSWER_TO_QUESTIONS: "yes"
+  SKIP_CHECK_REMOTE_IMAGE: "true"
+  SKIP_CI_IMAGE_CHECK: "true"
+  DB_RESET: "true"
+  VERBOSE: "true"
+  UPGRADE_TO_LATEST_CONSTRAINTS: ${{ github.event_name == 'push' || github.event_name == 'scheduled' }}
+  PYTHON_MAJOR_MINOR_VERSION: 3.6
+  USE_GITHUB_REGISTRY: "true"
+  CACHE_IMAGE_PREFIX: ${{ github.repository }}
+  CACHE_REGISTRY_USERNAME: ${{ github.actor }}
+  CACHE_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+
+jobs:
+
+  cancel-previous-workflow-run:
+    timeout-minutes: 60
+    name: "Cancel previous workflow run"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: potiuk/cancel-workflow-runs@v1
+        with:
+          workflow: quarantined.yml
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+  trigger-tests:
+    timeout-minutes: 5
+    name: "Checks if tests should be run"
+    runs-on: ubuntu-latest
+    needs: [cancel-previous-workflow-run]
+    outputs:
+      run-tests: ${{ steps.trigger-tests.outputs.run-tests }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: "Check if tests should be run"
+        run: "./scripts/ci/tools/ci_check_if_tests_should_be_run.sh"
+        id: trigger-tests
+
+  tests-quarantined:
+    timeout-minutes: 80
+    name: "Quarantined tests"
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    needs: [trigger-tests]
+    strategy:
+      matrix:
+        python-version: [3.6]
+        postgres-version: [9.6]
+      fail-fast: false
+    env:
+      BACKEND: postgres
+      PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
+      POSTGRES_VERSION: ${{ matrix.postgres-version }}
+      RUN_TESTS: "true"
+      TEST_TYPE: Quarantined
+      NUM_RUNS: 10
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    if: needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request'
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+      - name: "Set issue id for master"
+        if: github.ref == 'refs/heads/master'
+        run: |
+          echo "::set-env name=ISSUE_ID::86"
+      - name: "Set issue id for v1-10-stable"
+        if: github.ref == 'refs/heads/v1-10-stable'
+        run: |
+          echo "::set-env name=ISSUE_ID::10127"
+      - name: "Set issue id for v1-10-test"
+        if: github.ref == 'refs/heads/v1-10-test'
+        run: |
+          echo "::set-env name=ISSUE_ID::10128"
+      - name: "Free space"
+        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+      - name: "Build CI image ${{ matrix.python-version }}"
+        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
+      - name: "Tests"
+        run: ./scripts/ci/testing/ci_run_airflow_testing.sh
+      - uses: actions/upload-artifact@v2
+        name: Upload Quarantine test results
+        if: always()
+        with:
+          name: 'quarantined_tests'
+          path: 'files/test_result.xml'

CI.rst

@@ -67,8 +67,8 @@ The following components are part of the CI infrastructure
 CI run types
 ============
 
-The following CI Job runs are currently run for Apache Airflow, and each of the runs have different
-purpose and context.
+The following CI Job run types are currently run for Apache Airflow (run by ci.yaml workflow and
+quarantined.yaml workflows) and each of the run types have different purpose and context.
 
 Pull request run
 ----------------
@@ -126,7 +126,17 @@ DockerHub when pushing ``v1-10-stable`` manually.
 All runs consist of the same jobs, but the jobs behave slightly differently or they are skipped in different
 run categories. Here is a summary of the run categories with regards of the jobs they are running.
 Those jobs often have matrix run strategy which runs several different variations of the jobs
-(with different Backend type / Python version, type of the tests to run for example)
+(with different Backend type / Python version, type of the tests to run for example). The following chapter
+describes the workflows that execute for each run.
+
+Workflows
+=========
+
+CI Build Workflow
+-----------------
+
+This workflow is a regular workflow that performs the regular checks - none of the jobs should fail.
+The tests to run do not contain quarantined tests.
 
 +---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+
 | Job                       | Description                                                                                                    | Pull Request Run                   | Direct Push/Merge Run           | Scheduled Run                                                        |
@@ -148,8 +158,6 @@ Those jobs often have matrix run strategy which runs several different variation
 +---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+
 | Tests Kubernetes          | Run Kubernetes test                                                                                            | Yes (if tests-triggered)           | Yes                             | Yes *                                                                |
 +---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+
-| Quarantined tests         | Those are tests that are flaky and we need to fix them                                                         | Yes (if tests-triggered)           | Yes                             | Yes *                                                                |
-+---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+
 | Test OpenAPI client gen   | Tests if OpenAPIClient continues to generate                                                                   | Yes                                | Yes                             | Yes *                                                                |
 +---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+
 | Helm tests                | Runs tests for the Helm chart                                                                                  | Yes                                | Yes                             | Yes *                                                                |
@@ -164,3 +172,48 @@ Those jobs often have matrix run strategy which runs several different variation
 +---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+
 | Tag Repo nightly          | Tags the repository with nightly tagIt is a lightweight tag that moves nightly                                 | -                                  | -                               | Yes. Triggers DockerHub build for public registry                    |
 +---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+
+
+Quarantined build workflow
+--------------------------
+
+This workflow runs only quarantined tests. Those tests do not fail the build even if some tests fail (only if
+the whole pytest execution fails). Instead this workflow updates one of the issues where we keep status
+of quarantined tests. Once the test succeeds in NUM_RUNS subsequent runs, it is marked as stable and
+can be removed from quarantine. You can read more about quarantine in `<TESTING.rst>`_
+
+The issues are only updated if the test is run as direct push or scheduled run and only in the
+``apache/airflow`` repository - so that the issues are not updated in forks.
+
+The issues that gets updated are different for different branches:
+
+* master: `Quarantine tests master <https://github.com/apache/airflow/issues/10118>`_
+* v1-10-stable: `Quarantine tests v1-10-stable <https://github.com/apache/airflow/issues/10127>`_
+* v1-10-test: `Quarantine tests v1-10-test <hhttps://github.com/apache/airflow/issues/10128>`_
+
++---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+
+| Job                       | Description                                                                                                    | Pull Request Run                   | Direct Push/Merge Run           | Scheduled Run                                                        |
++===========================+================================================================================================================+====================================+=================================+======================================================================+
+| Cancel previous workflow  | Cancels the previously running workflow run if there is one running                                            | Yes                                | Yes                             | Yes *                                                                |
++---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+
+| Trigger tests             | Checks if tests should be triggered                                                                            | Yes                                | Yes                             | Yes *                                                                |
++---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+
+| Quarantined tests         | Those are tests that are flaky and we need to fix them                                                         | Yes (if tests-triggered)           | Yes (Updates quarantine issue)  | Yes * (updates quarantine issue)                                     |
++---------------------------+----------------------------------------------------------------------------------------------------------------+------------------------------------+---------------------------------+----------------------------------------------------------------------+
+
+Cancel other workflow runs workflow
+-----------------------------------
+
+This workflow is run only on schedule (every 5 minutes) it's only purpose is to cancel other running
+``CI Build`` workflows if important jobs failed in those runs. This is to save runners for other runs
+in case we know that the build will not succeed anyway without some basic fixes to static checks or
+documentation - effectively implementing missing "fail-fast" (on a job level) in Github Actions
+similar to fail-fast in matrix strategy.
+
+The jobs that are considered as "fail-fast" are:
+
+* Static checks
+* Docs
+* Prepare Backport packages
+* Helm tests
+* Build Prod Image
+* TTest OpenAPI client gen

scripts/ci/docker-compose/base.yml

@@ -39,6 +39,10 @@ services:
       - RUN_INTEGRATION_TESTS
       - ONLY_RUN_LONG_RUNNING_TESTS
       - ONLY_RUN_QUARANTINED_TESTS
+      - GITHUB_TOKEN
+      - GITHUB_REPOSITORY
+      - ISSUE_ID
+      - NUM_RUNS
       - BREEZE
       - INSTALL_AIRFLOW_VERSION
       - DB_RESET

scripts/ci/in_container/entrypoint_ci.sh

@@ -161,11 +161,12 @@ if [[ "${RUN_TESTS}" != "true" ]]; then
 fi
 set -u
 
+export RESULT_LOG_FILE="/files/test_result.xml"
+
 if [[ "${CI}" == "true" ]]; then
     EXTRA_PYTEST_ARGS=(
         "--verbosity=0"
         "--strict-markers"
-        "--instafail"
         "--durations=100"
         "--cov=airflow/"
         "--cov-config=.coveragerc"
@@ -174,6 +175,7 @@ if [[ "${CI}" == "true" ]]; then
         "--maxfail=50"
         "--pythonwarnings=ignore::DeprecationWarning"
         "--pythonwarnings=ignore::PendingDeprecationWarning"
+        "--junitxml=${RESULT_LOG_FILE}"
         )
 else
     EXTRA_PYTEST_ARGS=()
@@ -187,25 +189,43 @@ if [[ ${#@} -gt 0 && -n "$1" ]]; then
 fi
 
 if [[ -n ${RUN_INTEGRATION_TESTS:=""} ]]; then
+    # Integration tests
     for INT in ${RUN_INTEGRATION_TESTS}
     do
         EXTRA_PYTEST_ARGS+=("--integration" "${INT}")
     done
-    EXTRA_PYTEST_ARGS+=("-rpfExX")
+    EXTRA_PYTEST_ARGS+=(
+        # timeouts in seconds for individual tests
+        "--setup-timeout=20"
+        "--execution-timeout=60"
+        "--teardown-timeout=20"
+    )
+
 elif [[ ${ONLY_RUN_LONG_RUNNING_TESTS:=""} == "true" ]]; then
     EXTRA_PYTEST_ARGS+=(
         "-m" "long_running"
         "--include-long-running"
         "--verbosity=1"
-        "--reruns" "3"
-        "--timeout" "90")
+        "--setup-timeout=30"
+        "--execution-timeout=120"
+        "--teardown-timeout=30"
+    )
 elif [[ ${ONLY_RUN_QUARANTINED_TESTS:=""} == "true" ]]; then
     EXTRA_PYTEST_ARGS+=(
         "-m" "quarantined"
         "--include-quarantined"
         "--verbosity=1"
-        "--reruns" "3"
-        "--timeout" "90")
+        "--setup-timeout=10"
+        "--execution-timeout=50"
+        "--teardown-timeout=10"
+    )
+else
+    # Core tests
+    EXTRA_PYTEST_ARGS+=(
+        "--setup-timeout=10"
+        "--execution-timeout=30"
+        "--teardown-timeout=10"
+    )
 fi
 
 ARGS=("${EXTRA_PYTEST_ARGS[@]}" "${TESTS_TO_RUN[@]}")