From c5597d1fabe5d8f3a170885f6640344d93bf64bf Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 23 May 2023 00:19:38 +0200 Subject: [PATCH] Add Python 3.11 support (#27264) Add Python 3.11 support Python 3.11 has been released as scheduled on October 25, 2022 and finally, after all dependencies got upgraded - we can support it. --------- Co-authored-by: Tzu-ping Chung --- CI.rst | 8 +- LOCAL_VIRTUALENV.rst | 12 +- README.md | 18 +- airflow/__init__.py | 1 + airflow/providers/amazon/aws/hooks/dms.py | 2 +- airflow/providers/apache/hive/provider.yaml | 5 + .../apache/hive/transfers/mysql_to_hive.py | 36 ++- airflow/utils/log/file_task_handler.py | 2 +- dev/README_RELEASE_AIRFLOW.md | 2 +- .../src/airflow_breeze/global_constants.py | 4 +- dev/breeze/tests/test_cache.py | 4 +- dev/breeze/tests/test_selective_checks.py | 64 ++--- .../prepare_provider_packages.py | 2 +- dev/retag_docker_images.py | 2 +- .../howto/upgrading-from-1-10/index.rst | 2 +- .../installation/prerequisites.rst | 2 +- docs/docker-stack/build.rst | 16 +- generated/provider_dependencies.json | 240 ++++++++++++------ images/breeze/output-commands-hash.txt | 66 ++--- images/breeze/output-commands.svg | 100 ++++---- images/breeze/output_ci-image_build.svg | 4 +- images/breeze/output_ci-image_pull.svg | 60 +++-- images/breeze/output_ci-image_verify.svg | 32 ++- images/breeze/output_k8s_build-k8s-image.svg | 52 ++-- .../breeze/output_k8s_configure-cluster.svg | 60 +++-- images/breeze/output_k8s_create-cluster.svg | 4 +- images/breeze/output_k8s_delete-cluster.svg | 34 +-- images/breeze/output_k8s_deploy-airflow.svg | 70 ++--- images/breeze/output_k8s_k9s.svg | 32 ++- images/breeze/output_k8s_logs.svg | 34 +-- .../breeze/output_k8s_run-complete-tests.svg | 68 ++--- images/breeze/output_k8s_shell.svg | 40 +-- images/breeze/output_k8s_status.svg | 36 +-- images/breeze/output_k8s_tests.svg | 68 ++--- images/breeze/output_k8s_upload-k8s-image.svg | 58 +++-- images/breeze/output_prod-image_build.svg | 4 +- images/breeze/output_prod-image_pull.svg | 60 +++-- images/breeze/output_prod-image_verify.svg | 34 +-- ...elease-management_generate-constraints.svg | 4 +- ...release-management_release-prod-images.svg | 2 +- images/breeze/output_setup_config.svg | 2 +- images/breeze/output_shell.svg | 110 ++++---- images/breeze/output_start-airflow.svg | 2 +- .../output_testing_docker-compose-tests.svg | 2 +- .../output_testing_integration-tests.svg | 76 +++--- images/breeze/output_testing_tests.svg | 112 ++++---- ...re_commit_update_providers_dependencies.py | 18 +- setup.cfg | 1 + setup.py | 20 +- tests/dag_processing/test_processor.py | 98 ++++--- tests/decorators/test_python.py | 5 +- tests/models/test_param.py | 39 ++- .../providers/apache/hive/hooks/test_hive.py | 11 +- .../utils/test_mlengine_operator_utils.py | 16 +- 54 files changed, 1060 insertions(+), 796 deletions(-) diff --git a/CI.rst b/CI.rst index b0b5240423243..2484e8dc28360 100644 --- a/CI.rst +++ b/CI.rst @@ -59,10 +59,10 @@ Container Registry used as cache We are using GitHub Container Registry to store the results of the ``Build Images`` workflow which is used in the ``Tests`` workflow. -Currently in main version of Airflow we run tests in 4 different versions of Python (3.7, 3.8, 3.9, 3.10) -which means that we have to build 8 images (4 CI ones and 4 PROD ones). Yet we run around 12 jobs -with each of the CI images. That is a lot of time to just build the environment to run. Therefore -we are utilising ``pull_request_target`` feature of GitHub Actions. +Currently in main version of Airflow we run tests in all versions of Python supported, +which means that we have to build multiple images (one CI and one PROD for each Python version). +Yet we run many jobs (>15) - for each of the CI images. That is a lot of time to just build the +environment to run. Therefore we are utilising ``pull_request_target`` feature of GitHub Actions. This feature allows to run a separate, independent workflow, when the main workflow is run - this separate workflow is different than the main one, because by default it runs using ``main`` version diff --git a/LOCAL_VIRTUALENV.rst b/LOCAL_VIRTUALENV.rst index e52c529094454..00205215b196c 100644 --- a/LOCAL_VIRTUALENV.rst +++ b/LOCAL_VIRTUALENV.rst @@ -51,7 +51,7 @@ Required Software Packages Use system-level package managers like yum, apt-get for Linux, or Homebrew for macOS to install required software packages: -* Python (One of: 3.7, 3.8, 3.9, 3.10) +* Python (One of: 3.7, 3.8, 3.9, 3.10, 3.11) * MySQL 5.7+ * libxml @@ -102,7 +102,7 @@ Creating a Local virtualenv To use your IDE for Airflow development and testing, you need to configure a virtual environment. Ideally you should set up virtualenv for all Python versions that Airflow -supports (3.7, 3.8, 3.9, 3.10). +supports (3.7, 3.8, 3.9, 3.10, 3.11). To create and initialize the local virtualenv: @@ -122,7 +122,7 @@ To create and initialize the local virtualenv: .. code-block:: bash - conda create -n airflow python=3.7 # or 3.8, 3.9, 3.10 + conda create -n airflow python=3.7 # or 3.8, 3.9, 3.10, 3.11 conda activate airflow 2. Install Python PIP requirements: @@ -150,7 +150,7 @@ for different python versions). For development on current main source: .. code-block:: bash - # use the same version of python as you are working with, 3.7, 3.8, 3.9, or 3.10 + # use the same version of python as you are working with, 3.7, 3.8, 3.9, 3.10 or 3.11 pip install -e ".[devel,]" \ --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-source-providers-3.7.txt" @@ -163,7 +163,7 @@ You can also install Airflow in non-editable mode: .. code-block:: bash - # use the same version of python as you are working with, 3.7, 3.8, 3.9, or 3.10 + # use the same version of python as you are working with, 3.7, 3.8, 3.9, 3.10 or 3.11 pip install ".[devel,]" \ --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-source-providers-3.7.txt" @@ -173,7 +173,7 @@ sources, unless you set ``INSTALL_PROVIDERS_FROM_SOURCES`` environment variable .. code-block:: bash - # use the same version of python as you are working with, 3.7, 3.8, 3.9, or 3.10 + # use the same version of python as you are working with, 3.7, 3.8, 3.9, 3.10 or 3.11 INSTALL_PROVIDERS_FROM_SOURCES="true" pip install ".[devel,]" \ --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-source-providers-3.7.txt" diff --git a/README.md b/README.md index a9bc5aed7ee9e..52c97fe1f97e4 100644 --- a/README.md +++ b/README.md @@ -86,15 +86,15 @@ Airflow is not a streaming solution, but it is often used to process real-time d Apache Airflow is tested with: -| | Main version (dev) | Stable version (2.6.1) | -|------------|------------------------------|------------------------| -| Python | 3.7, 3.8, 3.9, 3.10 | 3.7, 3.8, 3.9, 3.10 | -| Platform | AMD64/ARM64(\*) | AMD64/ARM64(\*) | -| Kubernetes | 1.23, 1.24, 1.25, 1.26, 1.27 | 1.23, 1.24, 1.25, 1.26 | -| PostgreSQL | 11, 12, 13, 14, 15 | 11, 12, 13, 14, 15 | -| MySQL | 5.7, 8 | 5.7, 8 | -| SQLite | 3.15.0+ | 3.15.0+ | -| MSSQL | 2017(\*), 2019(\*) | 2017(\*), 2019(\*) | +| | Main version (dev) | Stable version (2.6.1) | +|------------|------------------------------|---------------------------| +| Python | 3.7, 3.8, 3.9, 3.10, 3.11 | 3.7, 3.8, 3.9, 3.10, 3.11 | +| Platform | AMD64/ARM64(\*) | AMD64/ARM64(\*) | +| Kubernetes | 1.23, 1.24, 1.25, 1.26, 1.27 | 1.23, 1.24, 1.25, 1.26 | +| PostgreSQL | 11, 12, 13, 14, 15 | 11, 12, 13, 14, 15 | +| MySQL | 5.7, 8 | 5.7, 8 | +| SQLite | 3.15.0+ | 3.15.0+ | +| MSSQL | 2017(\*), 2019(\*) | 2017(\*), 2019(\*) | \* Experimental diff --git a/airflow/__init__.py b/airflow/__init__.py index cf5622af69b5a..f0174e5d59f49 100644 --- a/airflow/__init__.py +++ b/airflow/__init__.py @@ -72,6 +72,7 @@ PY38 = sys.version_info >= (3, 8) PY39 = sys.version_info >= (3, 9) PY310 = sys.version_info >= (3, 10) +PY311 = sys.version_info >= (3, 11) # Things to lazy import in form {local_name: ('target_module', 'target_name')} __lazy_imports: dict[str, tuple[str, str]] = { diff --git a/airflow/providers/amazon/aws/hooks/dms.py b/airflow/providers/amazon/aws/hooks/dms.py index 94cb5ac8ba2eb..a8ab6bf0a0dd6 100644 --- a/airflow/providers/amazon/aws/hooks/dms.py +++ b/airflow/providers/amazon/aws/hooks/dms.py @@ -207,7 +207,7 @@ def wait_for_task_status(self, replication_task_arn: str, status: DmsTaskWaiterS raise TypeError("Status must be an instance of DmsTaskWaiterStatus") dms_client = self.get_conn() - waiter = dms_client.get_waiter(f"replication_task_{status}") + waiter = dms_client.get_waiter(f"replication_task_{status.value}") waiter.wait( Filters=[ { diff --git a/airflow/providers/apache/hive/provider.yaml b/airflow/providers/apache/hive/provider.yaml index 0ec7e0959ba25..3c16e1d8843c3 100644 --- a/airflow/providers/apache/hive/provider.yaml +++ b/airflow/providers/apache/hive/provider.yaml @@ -63,6 +63,11 @@ dependencies: - sasl>=0.3.1; python_version>="3.9" - thrift>=0.9.2 +# Excluded because python-sasl is not yet compatible +# with 3.11. See https://github.com/cloudera/python-sasl/issues/30 +excluded-python-versions: + - "3.11" + integrations: - integration-name: Apache Hive external-doc-url: https://hive.apache.org/ diff --git a/airflow/providers/apache/hive/transfers/mysql_to_hive.py b/airflow/providers/apache/hive/transfers/mysql_to_hive.py index 5d042bf921393..639723611ed72 100644 --- a/airflow/providers/apache/hive/transfers/mysql_to_hive.py +++ b/airflow/providers/apache/hive/transfers/mysql_to_hive.py @@ -19,6 +19,7 @@ from __future__ import annotations from collections import OrderedDict +from contextlib import closing from tempfile import NamedTemporaryFile from typing import TYPE_CHECKING, Sequence @@ -131,28 +132,25 @@ def type_map(cls, mysql_type: int) -> str: def execute(self, context: Context): hive = HiveCliHook(hive_cli_conn_id=self.hive_cli_conn_id, auth=self.hive_auth) mysql = MySqlHook(mysql_conn_id=self.mysql_conn_id) - self.log.info("Dumping MySQL query results to local file") - conn = mysql.get_conn() - cursor = conn.cursor() - cursor.execute(self.sql) with NamedTemporaryFile("wb") as f: - csv_writer = csv.writer( - f, - delimiter=self.delimiter, - quoting=self.quoting, - quotechar=self.quotechar, - escapechar=self.escapechar, - encoding="utf-8", - ) - field_dict = OrderedDict() - if cursor.description is not None: - for field in cursor.description: - field_dict[field[0]] = self.type_map(field[1]) - csv_writer.writerows(cursor) + with closing(mysql.get_conn()) as conn: + with closing(conn.cursor()) as cursor: + cursor.execute(self.sql) + csv_writer = csv.writer( + f, + delimiter=self.delimiter, + quoting=self.quoting, + quotechar=self.quotechar if self.quoting != csv.QUOTE_NONE else None, + escapechar=self.escapechar, + encoding="utf-8", + ) + field_dict = OrderedDict() + if cursor.description is not None: + for field in cursor.description: + field_dict[field[0]] = self.type_map(field[1]) + csv_writer.writerows(cursor) f.flush() - cursor.close() - conn.close() # type: ignore[misc] self.log.info("Loading file into Hive") hive.load_file( f.name, diff --git a/airflow/utils/log/file_task_handler.py b/airflow/utils/log/file_task_handler.py index 28d8c628ccec9..194045c62635a 100644 --- a/airflow/utils/log/file_task_handler.py +++ b/airflow/utils/log/file_task_handler.py @@ -201,7 +201,7 @@ def add_triggerer_suffix(full_path, job_id=None): triggerer instances. """ full_path = Path(full_path).as_posix() - full_path += f".{LogType.TRIGGER}" + full_path += f".{LogType.TRIGGER.value}" if job_id: full_path += f".{job_id}.log" return full_path diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index 6d42b592083f7..7e10bcfe0ce58 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -670,7 +670,7 @@ the older branches, you should set the "skip" field to true. ## Verify production images ```shell script -for PYTHON in 3.7 3.8 3.9 3.10 +for PYTHON in 3.7 3.8 3.9 3.10 3.11 do docker pull apache/airflow:${VERSION}-python${PYTHON} breeze prod-image verify --image-name apache/airflow:${VERSION}-python${PYTHON} diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index eec14c2951ca0..a72f6b7513831 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -36,7 +36,7 @@ APACHE_AIRFLOW_GITHUB_REPOSITORY = "apache/airflow" # Checked before putting in build cache -ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] DEFAULT_PYTHON_MAJOR_MINOR_VERSION = ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS[0] ALLOWED_ARCHITECTURES = [Architecture.X86_64, Architecture.ARM] ALLOWED_BACKENDS = ["sqlite", "mysql", "postgres", "mssql"] @@ -174,7 +174,7 @@ def get_default_platform_machine() -> str: PYTHONDONTWRITEBYTECODE = True PRODUCTION_IMAGE = False -ALL_PYTHON_MAJOR_MINOR_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +ALL_PYTHON_MAJOR_MINOR_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] CURRENT_PYTHON_MAJOR_MINOR_VERSIONS = ALL_PYTHON_MAJOR_MINOR_VERSIONS CURRENT_POSTGRES_VERSIONS = ["11", "12", "13", "14", "15"] DEFAULT_POSTGRES_VERSION = CURRENT_POSTGRES_VERSIONS[0] diff --git a/dev/breeze/tests/test_cache.py b/dev/breeze/tests/test_cache.py index a37a91165b4fc..1db27494abd07 100644 --- a/dev/breeze/tests/test_cache.py +++ b/dev/breeze/tests/test_cache.py @@ -36,8 +36,8 @@ [ ("backend", "mysql", (True, ["sqlite", "mysql", "postgres", "mssql"]), None), ("backend", "xxx", (False, ["sqlite", "mysql", "postgres", "mssql"]), None), - ("python_major_minor_version", "3.8", (True, ["3.7", "3.8", "3.9", "3.10"]), None), - ("python_major_minor_version", "3.5", (False, ["3.7", "3.8", "3.9", "3.10"]), None), + ("python_major_minor_version", "3.8", (True, ["3.7", "3.8", "3.9", "3.10", "3.11"]), None), + ("python_major_minor_version", "3.5", (False, ["3.7", "3.8", "3.9", "3.10", "3.11"]), None), ("missing", "value", None, AttributeError), ], ) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 4ac7adb34d286..7a23c3c551e65 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -266,10 +266,10 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ("setup.py",), { "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", - "python-versions": "['3.7', '3.8', '3.9', '3.10']", - "python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", + "python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", @@ -289,10 +289,10 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ("generated/provider_dependencies.json",), { "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", - "python-versions": "['3.7', '3.8', '3.9', '3.10']", - "python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", + "python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", @@ -397,10 +397,10 @@ def test_expected_output_pull_request_main( "main", { "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", - "python-versions": "['3.7', '3.8', '3.9', '3.10']", - "python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", + "python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "run-tests": "true", "docs-build": "true", @@ -424,10 +424,10 @@ def test_expected_output_pull_request_main( "main", { "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", - "python-versions": "['3.7', '3.8', '3.9', '3.10']", - "python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", + "python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "run-tests": "true", "docs-build": "true", @@ -449,10 +449,10 @@ def test_expected_output_pull_request_main( "main", { "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", - "python-versions": "['3.7', '3.8', '3.9', '3.10']", - "python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", + "python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "run-tests": "true", "docs-build": "true", @@ -474,10 +474,10 @@ def test_expected_output_pull_request_main( "v2-3-stable", { "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", - "python-versions": "['3.7', '3.8', '3.9', '3.10']", - "python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", + "python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "run-tests": "true", "docs-build": "true", @@ -787,8 +787,8 @@ def test_expected_output_pull_request_target( "main", { "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", @@ -807,8 +807,8 @@ def test_expected_output_pull_request_target( "v2-3-stable", { "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "needs-helm-tests": "false", "run-tests": "true", @@ -826,8 +826,8 @@ def test_expected_output_pull_request_target( "main", { "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED, - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", @@ -878,8 +878,8 @@ def test_no_commit_provided_trigger_full_build_for_any_event_type(github_event): ) assert_outputs_are_printed( { - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py index a2b7787f490ca..f78f53a122d14 100755 --- a/dev/provider_packages/prepare_provider_packages.py +++ b/dev/provider_packages/prepare_provider_packages.py @@ -52,7 +52,7 @@ from rich.syntax import Syntax from yaml import safe_load -ALL_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +ALL_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] MIN_AIRFLOW_VERSION = "2.4.0" # In case you have some providers that you want to have different min-airflow version for, diff --git a/dev/retag_docker_images.py b/dev/retag_docker_images.py index 8c9f6eb78d386..ecc304eaa8b4c 100755 --- a/dev/retag_docker_images.py +++ b/dev/retag_docker_images.py @@ -31,7 +31,7 @@ import rich_click as click -PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] GHCR_IO_PREFIX = "ghcr.io" diff --git a/docs/apache-airflow/howto/upgrading-from-1-10/index.rst b/docs/apache-airflow/howto/upgrading-from-1-10/index.rst index c599f357f7c30..66a6cccb927b0 100644 --- a/docs/apache-airflow/howto/upgrading-from-1-10/index.rst +++ b/docs/apache-airflow/howto/upgrading-from-1-10/index.rst @@ -34,7 +34,7 @@ Airflow 1.10 was the last release series to support Python 2. Airflow 2.0.0 requires Python 3.6+ and has been tested with Python versions 3.6, 3.7 and 3.8. Python 3.9 support was added from Airflow 2.1.2. -Airflow 2.3.0 dropped support for Python 3.6. It's tested with Python 3.7, 3.8, 3.9 and 3.10. +Airflow 2.3.0 dropped support for Python 3.6. It's tested with Python 3.7, 3.8, 3.9, 3.10, 3.11. If you have a specific task that still requires Python 2 then you can use the ``@task.virtualenv``, ``@task.docker`` or ``@task.kubernetes`` decorators for this. diff --git a/docs/apache-airflow/installation/prerequisites.rst b/docs/apache-airflow/installation/prerequisites.rst index 82a0310721e3f..d10650bad0563 100644 --- a/docs/apache-airflow/installation/prerequisites.rst +++ b/docs/apache-airflow/installation/prerequisites.rst @@ -20,7 +20,7 @@ Prerequisites Starting with Airflow 2.3.0, Airflow is tested with:. -* Python: 3.7, 3.8, 3.9, 3.10 +* Python: 3.7, 3.8, 3.9, 3.10, 3.11 * Databases: diff --git a/docs/docker-stack/build.rst b/docs/docker-stack/build.rst index 7beed0afed6b0..56edb3837d6dd 100644 --- a/docs/docker-stack/build.rst +++ b/docs/docker-stack/build.rst @@ -288,14 +288,14 @@ There are two types of images you can extend your image from: Naming conventions for the images: -+----------------+------------------+---------------------------------+--------------------------------------+ -| Image | Python | Standard image | Slim image | -+================+==================+=================================+======================================+ -| Latest default | 3.7 | apache/airflow:latest | apache/airflow:slim-latest | -| Default | 3.7 | apache/airflow:X.Y.Z | apache/airflow:slim-X.Y.Z | -| Latest | 3.7,3.8,3.9,3.10 | apache/airflow:latest-pythonN.M | apache/airflow:slim-latest-pythonN.M | -| Specific | 3.7,3.8,3.9,3.10 | apache/airflow:X.Y.Z-pythonN.M | apache/airflow:slim-X.Y.Z-pythonN.M | -+----------------+------------------+---------------------------------+--------------------------------------+ ++----------------+-----------------------+---------------------------------+--------------------------------------+ +| Image | Python | Standard image | Slim image | ++================+=======================+=================================+======================================+ +| Latest default | 3.7 | apache/airflow:latest | apache/airflow:slim-latest | +| Default | 3.7 | apache/airflow:X.Y.Z | apache/airflow:slim-X.Y.Z | +| Latest | 3.7,3.8,3.9,3.10,3.11 | apache/airflow:latest-pythonN.M | apache/airflow:slim-latest-pythonN.M | +| Specific | 3.7,3.8,3.9,3.10,3.11 | apache/airflow:X.Y.Z-pythonN.M | apache/airflow:slim-X.Y.Z-pythonN.M | ++----------------+-----------------------+---------------------------------+--------------------------------------+ * The "latest" image is always the latest released stable version available. diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 0d836769f4d61..ae286f97e3382 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -6,14 +6,16 @@ ], "cross-providers-deps": [ "http" - ] + ], + "excluded-python-versions": [] }, "alibaba": { "deps": [ "apache-airflow>=2.4.0", "oss2>=2.14.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "amazon": { "deps": [ @@ -41,7 +43,8 @@ "mongo", "salesforce", "ssh" - ] + ], + "excluded-python-versions": [] }, "apache.beam": { "deps": [ @@ -50,14 +53,16 @@ ], "cross-providers-deps": [ "google" - ] + ], + "excluded-python-versions": [] }, "apache.cassandra": { "deps": [ "apache-airflow>=2.4.0", "cassandra-driver>=3.13.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "apache.drill": { "deps": [ @@ -67,7 +72,8 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "apache.druid": { "deps": [ @@ -78,7 +84,8 @@ "cross-providers-deps": [ "apache.hive", "common.sql" - ] + ], + "excluded-python-versions": [] }, "apache.flink": { "deps": [ @@ -88,14 +95,16 @@ ], "cross-providers-deps": [ "cncf.kubernetes" - ] + ], + "excluded-python-versions": [] }, "apache.hdfs": { "deps": [ "apache-airflow>=2.4.0", "hdfs[avro,dataframe,kerberos]>=2.0.4" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "apache.hive": { "deps": [ @@ -115,6 +124,9 @@ "presto", "samba", "vertica" + ], + "excluded-python-versions": [ + "3.11" ] }, "apache.impala": { @@ -124,7 +136,8 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "apache.kafka": { "deps": [ @@ -132,14 +145,16 @@ "asgiref", "confluent-kafka>=1.8.2" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "apache.kylin": { "deps": [ "apache-airflow>=2.4.0", "kylinpy>=2.6" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "apache.livy": { "deps": [ @@ -150,13 +165,15 @@ ], "cross-providers-deps": [ "http" - ] + ], + "excluded-python-versions": [] }, "apache.pig": { "deps": [ "apache-airflow>=2.4.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "apache.pinot": { "deps": [ @@ -166,41 +183,47 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "apache.spark": { "deps": [ "apache-airflow>=2.4.0", "pyspark" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "apache.sqoop": { "deps": [ "apache-airflow>=2.4.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "arangodb": { "deps": [ "apache-airflow>=2.4.0", "python-arango>=7.3.2" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "asana": { "deps": [ "apache-airflow>=2.4.0", "asana>=0.10" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "atlassian.jira": { "deps": [ "apache-airflow>=2.4.0", "atlassian-python-api>=1.14.2" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "celery": { "deps": [ @@ -208,14 +231,16 @@ "celery>=5.2.3,<6", "flower>=1.0.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "cloudant": { "deps": [ "apache-airflow>=2.4.0", "cloudant>=2.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "cncf.kubernetes": { "deps": [ @@ -225,13 +250,15 @@ "kubernetes>=21.7.0,<24", "kubernetes_asyncio>=18.20.1,<25" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "common.sql": { "deps": [ "sqlparse>=0.4.2" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "databricks": { "deps": [ @@ -243,14 +270,16 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "datadog": { "deps": [ "apache-airflow>=2.4.0", "datadog>=0.14.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "dbt.cloud": { "deps": [ @@ -261,7 +290,8 @@ ], "cross-providers-deps": [ "http" - ] + ], + "excluded-python-versions": [] }, "dingding": { "deps": [ @@ -270,7 +300,8 @@ ], "cross-providers-deps": [ "http" - ] + ], + "excluded-python-versions": [] }, "discord": { "deps": [ @@ -279,7 +310,8 @@ ], "cross-providers-deps": [ "http" - ] + ], + "excluded-python-versions": [] }, "docker": { "deps": [ @@ -287,7 +319,8 @@ "docker>=5.0.3", "python-dotenv>=0.21.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "elasticsearch": { "deps": [ @@ -299,7 +332,8 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "exasol": { "deps": [ @@ -310,25 +344,29 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "facebook": { "deps": [ "apache-airflow>=2.4.0", "facebook-business>=6.0.2" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "ftp": { "deps": [], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "github": { "deps": [ "PyGithub!=1.58", "apache-airflow>=2.4.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "google": { "deps": [ @@ -403,7 +441,8 @@ "sftp", "ssh", "trino" - ] + ], + "excluded-python-versions": [] }, "grpc": { "deps": [ @@ -412,7 +451,8 @@ "google-auth>=1.0.0, <3.0.0", "grpcio>=1.15.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "hashicorp": { "deps": [ @@ -421,7 +461,8 @@ ], "cross-providers-deps": [ "google" - ] + ], + "excluded-python-versions": [] }, "http": { "deps": [ @@ -430,11 +471,13 @@ "requests>=2.26.0", "requests_toolbelt" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "imap": { "deps": [], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "influxdb": { "deps": [ @@ -442,7 +485,8 @@ "influxdb-client>=1.19.0", "requests>=2.26.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "jdbc": { "deps": [ @@ -452,14 +496,16 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "jenkins": { "deps": [ "apache-airflow>=2.4.0", "python-jenkins>=1.0.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "microsoft.azure": { "deps": [ @@ -486,7 +532,8 @@ "google", "oracle", "sftp" - ] + ], + "excluded-python-versions": [] }, "microsoft.mssql": { "deps": [ @@ -496,21 +543,24 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "microsoft.psrp": { "deps": [ "apache-airflow>=2.4.0", "pypsrp>=0.8.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "microsoft.winrm": { "deps": [ "apache-airflow>=2.4.0", "pywinrm>=0.4" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "mongo": { "deps": [ @@ -518,7 +568,8 @@ "dnspython>=1.13.0", "pymongo>=3.6.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "mysql": { "deps": [ @@ -532,14 +583,16 @@ "presto", "trino", "vertica" - ] + ], + "excluded-python-versions": [] }, "neo4j": { "deps": [ "apache-airflow>=2.4.0", "neo4j>=4.2.1" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "odbc": { "deps": [ @@ -549,13 +602,15 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "openfaas": { "deps": [ "apache-airflow>=2.4.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "openlineage": { "deps": [ @@ -565,14 +620,16 @@ "openlineage-integration-common>=0.22.0", "openlineage-python>=0.22.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "opsgenie": { "deps": [ "apache-airflow>=2.4.0", "opsgenie-sdk>=2.1.5" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "oracle": { "deps": [ @@ -582,14 +639,16 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "pagerduty": { "deps": [ "apache-airflow>=2.4.0", "pdpyras>=4.1.2" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "papermill": { "deps": [ @@ -597,14 +656,16 @@ "papermill[all]>=1.2.1", "scrapbook[all]" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "plexus": { "deps": [ "apache-airflow>=2.4.0", "arrow>=0.16.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "postgres": { "deps": [ @@ -615,7 +676,8 @@ "cross-providers-deps": [ "amazon", "common.sql" - ] + ], + "excluded-python-versions": [] }, "presto": { "deps": [ @@ -627,7 +689,8 @@ "cross-providers-deps": [ "common.sql", "google" - ] + ], + "excluded-python-versions": [] }, "qubole": { "deps": [ @@ -637,14 +700,16 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "redis": { "deps": [ "apache-airflow>=2.4.0", "redis>=3.2.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "salesforce": { "deps": [ @@ -652,28 +717,32 @@ "pandas>=0.17.1", "simple-salesforce>=1.0.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "samba": { "deps": [ "apache-airflow>=2.4.0", "smbprotocol>=1.5.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "segment": { "deps": [ "analytics-python>=1.2.9", "apache-airflow>=2.4.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "sendgrid": { "deps": [ "apache-airflow>=2.4.0", "sendgrid>=6.0.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "sftp": { "deps": [ @@ -682,14 +751,16 @@ ], "cross-providers-deps": [ "ssh" - ] + ], + "excluded-python-versions": [] }, "singularity": { "deps": [ "apache-airflow>=2.4.0", "spython>=0.0.56" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "slack": { "deps": [ @@ -699,13 +770,15 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "smtp": { "deps": [ "apache-airflow>=2.4.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "snowflake": { "deps": [ @@ -717,7 +790,8 @@ "cross-providers-deps": [ "common.sql", "slack" - ] + ], + "excluded-python-versions": [] }, "sqlite": { "deps": [ @@ -725,7 +799,8 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "ssh": { "deps": [ @@ -733,27 +808,31 @@ "paramiko>=2.6.0", "sshtunnel>=0.3.2" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "tableau": { "deps": [ "apache-airflow>=2.4.0", "tableauserverclient" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "tabular": { "deps": [ "apache-airflow>=2.4.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "telegram": { "deps": [ "apache-airflow>=2.4.0", "python-telegram-bot>=20.0.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "trino": { "deps": [ @@ -765,7 +844,8 @@ "cross-providers-deps": [ "common.sql", "google" - ] + ], + "excluded-python-versions": [] }, "vertica": { "deps": [ @@ -775,13 +855,15 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "zendesk": { "deps": [ "apache-airflow>=2.4.0", "zenpy>=2.0.24" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] } } diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 519db1894e68e..79046e5eb0e78 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -1,7 +1,7 @@ # This file is automatically generated by pre-commit. If you have a conflict with this file # Please do not solve it but run `breeze setup regenerate-command-images`. # This command should fix the conflict and regenerate help images that you have conflict with. -main:3b0efd589fb61236e9fb1e4422de78c9 +main:5bd7ecf7e8fdd34879f24ad8ac390405 build-docs:3b89efaf5551b1782227cd382c019990 ci:fix-ownership:fee2c9ec9ef19686792002ae054fecdd ci:free-space:47234aa0a60b0efd84972e6e797379f8 @@ -9,56 +9,56 @@ ci:get-workflow-info:01ee34c33ad62fa5dc33e0ac8773223f ci:resource-check:1d4fe47dff9fc64ac1648ec4beb2d85c ci:selective-check:8a39978ee69d496dae2533d37a48b137 ci:2868dbcdd482663e9d6ccd00055b9cac -ci-image:build:3ffe4dd24ae7090415543e27d8504955 -ci-image:pull:c16c6e57c748bfe9b365b4ffafb18472 -ci-image:verify:aee88f55e8837028d19316356e29b009 -ci-image:d53ee5067c46a6d8f3b21d0098c56ca2 +ci-image:build:4c7702285269907eaa7d48ab7fd242e3 +ci-image:pull:68fb55a81db9fc1615424c9bef438bd7 +ci-image:verify:3c2588eddc2bef7dd14c142a1f56a56f +ci-image:587a537fed951c401d2a3cd82ce1b3de cleanup:231de69d5f47ba29c883164e4575e310 compile-www-assets:c8a8c4f002f7246d0541897fc7c70313 down:99efb2055475cd40656d5498aee74ef6 exec:42bbd3c1659128b0341ae118c3482da2 -k8s:build-k8s-image:bf48ee8742757dc19a4c3fad23f060a7 -k8s:configure-cluster:5ee4c2f1f259f162041c33553272559b -k8s:create-cluster:7feaa0211b2b37bc41bd2fb40693bb7b -k8s:delete-cluster:e7f0a33c8c4f5e26e4c1bac0dd71211f -k8s:deploy-airflow:771ec6bb40959ae3a0dbc677ddb4725f -k8s:k9s:9f9827b6ca7a9c92e17bd2dabd6a4563 -k8s:logs:f30cffb7efb6272db0592a46dea2f128 -k8s:run-complete-tests:bc202754890cf4081dc7d3c92965b9ab +k8s:build-k8s-image:7ef9f8d1fa725b7d6dc7daad72d66825 +k8s:configure-cluster:1c64a309a58dea6baf61a5ed223f1ac5 +k8s:create-cluster:49b63399485b06ffe5424c737a1e8ab3 +k8s:delete-cluster:0741154a3988a3c8591ff9af409ddf10 +k8s:deploy-airflow:1e5038d63fe6b77f017fb47ab470a860 +k8s:k9s:e2ffb041b003faabfafe2aff3b48f023 +k8s:logs:89503ad44dc0b480146ca2978719f585 +k8s:run-complete-tests:6adc95e5f48cee11aebeac6bcd3c7e7f k8s:setup-env:b0ea72ef1819f831b1f80e8bd4d299ce -k8s:shell:d9d2adfac4fcb1b6a2854fe115c34bfe -k8s:status:455ee4ce95a02b7ca375af0f1491cccb -k8s:tests:98733962051b223b6670dcd5dbe58bfd -k8s:upload-k8s-image:07980a9e7c71f26025103e631d66f73e -k8s:d9dd78c36e68405c06721a0e9d4a3f6c -prod-image:build:a1608071b5a80b3cc442da9d00e1525b -prod-image:pull:e3c89dd908fc44adf6e159c2950ebdd0 -prod-image:verify:31bc5efada1d70a0a31990025db1a093 -prod-image:79bd4cc9de03ab7e1d75f025d75eee46 +k8s:shell:f9f4a460a0b94df5d28c7522a19f76b8 +k8s:status:f4a5a7c657825ce6ab76e7d9152d98f4 +k8s:tests:3ae800a20447049eafdc10db122a10bf +k8s:upload-k8s-image:aa691f32e5963fe7a0435ef2ca8b59ae +k8s:aa5348c18c987567bbd7800fc2aff378 +prod-image:build:4e0aac2327f8e068c0db70ecdd11dfa3 +prod-image:pull:722a843b29ca251aed1a5e001cf48db2 +prod-image:verify:4edd26d1126761777e934eae2b8d57dd +prod-image:9ff9f57f7b7ff65718e95f5468cdb615 release-management:create-minor-branch:6a01066dce15e09fb269a8385626657c -release-management:generate-constraints:ae30d6ad49a1b2c15b61cb29080fd957 +release-management:generate-constraints:0c8e5c1b57b486724760d4afdfd571f7 release-management:generate-issue-content-providers:421c1b186818a6251c16f7f3b7807292 release-management:install-provider-packages:5838b06b78e3c5c6e8380024867a1a8d release-management:prepare-airflow-package:3ac14ea6d2b09614959c0ec4fd564789 release-management:prepare-provider-documentation:13f5fa922825a922c3525b5a57b2a80c release-management:prepare-provider-packages:cf41c33c6d6121efef1f1d97333e8710 -release-management:release-prod-images:c9bc40938e0efad49e51ef66e83f9527 +release-management:release-prod-images:dff5d068a253dcc3ac1e65a6af150e3d release-management:start-rc-process:6aafbaceabd7b67b9a1af4c2f59abc4c release-management:start-release:acb384d86e02ff5fde1bf971897be17c release-management:verify-provider-packages:566c60fb1bfdc5ed7c4be590736891b2 -release-management:c0ecc3e5cd4a9f1aac18a9d6eb1ddfb6 +release-management:dbf5a361d4e573c4dcf730237b2351b0 setup:autocomplete:03343478bf1d0cf9c101d454cdb63b68 setup:check-all-params-in-groups:1a6a07dcef12a55bf42bcbcef0d86adc -setup:config:3ffcd35dd24b486ddf1d08b797e3d017 +setup:config:584023d1d225879a5d19e347fe9db1d3 setup:regenerate-command-images:873006338375c95a3d683da214366191 setup:self-upgrade:d02f70c7a230eae3463ceec2056b63fa setup:version:123b462a421884dc2320ffc5e54b2478 -setup:f576a6309f2be7a4507304eb7700ad82 -shell:bd3e004a92ebcec8feb40fc5cd95872d -start-airflow:ee5066f1420a489864b48bc4e5e472da +setup:df504f6ad27442c86f9afaa8f82c2eef +shell:e566b8314bc7fee8580afdbacb55cd3a +start-airflow:3bf24a94653fa9236a70a94a499b5b7c static-checks:9985d1db64592e29ab71b8a000ce302e -testing:docker-compose-tests:183435da5d5df96f1bec016fcb5d4c76 +testing:docker-compose-tests:fb26a48e083b0a5c6eae2ebd052eaf72 testing:helm-tests:936cf28fd84ce4ff5113795fdae9624b -testing:integration-tests:6b302f81c1038215bbc9f22f9490fcfc -testing:tests:9a8e3645ff5341788b5cd683b1c4bdcd -testing:3ccba105be3eda9449bc446df4b34d2c +testing:integration-tests:372b0e421ebcefb1ea7a1008658134c3 +testing:tests:7ea8d4eaaa62e5ae32fe636b54d96903 +testing:fe3c5b6f3c634e29be1302580ffe686f diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index ba821bd664d4e..c6642829144b5 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -1,4 +1,4 @@ - + - + @@ -177,9 +177,12 @@ + + + - Breeze commands + Breeze commands @@ -190,50 +193,51 @@ -Usage: breeze [OPTIONS] COMMAND [ARGS]... +Usage: breeze [OPTIONS] COMMAND [ARGS]... -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                            │ -│(all | all-testable | cassandra | celery | kafka | kerberos | mongo | otel | pinot |     │ -│statsd | statsd | trino)                                                                 │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ -│(INTEGER RANGE)                                                                       │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Basic developer commands ───────────────────────────────────────────────────────────────────────────────────────────╮ -│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ -│if contents of www directory changed.                                                           │ -│static-checks     Run static checks.                                                                              │ -│build-docs        Build documentation in the container.                                                           │ -│down              Stop running breeze environment.                                                                │ -│shell             Enter breeze environment. this is the default command use when no other is selected.            │ -│exec              Joins the interactive shell of running airflow container.                                       │ -│compile-www-assetsCompiles www assets.                                                                            │ -│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced command groups ────────────────────────────────────────────────────────────────────────────────────────────╮ -│testing                Tools that developers can use to run tests                                                 │ -│ci-image               Tools that developers can use to manually manage CI images                                 │ -│k8s                    Tools that developers use to run Kubernetes tests                                          │ -│prod-image             Tools that developers can use to manually manage PROD images                               │ -│setup                  Tools that developers can use to configure Breeze                                          │ -│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ -│ci                     Tools that CI workflows use to cleanup/manage CI environment                               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ +│[default: 3.7]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                            │ +│(all | all-testable | cassandra | celery | kafka | kerberos | mongo | otel | pinot |     │ +│statsd | statsd | trino)                                                                 │ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ +│(INTEGER RANGE)                                                                       │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic developer commands ───────────────────────────────────────────────────────────────────────────────────────────╮ +│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ +│if contents of www directory changed.                                                           │ +│static-checks     Run static checks.                                                                              │ +│build-docs        Build documentation in the container.                                                           │ +│down              Stop running breeze environment.                                                                │ +│shell             Enter breeze environment. this is the default command use when no other is selected.            │ +│exec              Joins the interactive shell of running airflow container.                                       │ +│compile-www-assetsCompiles www assets.                                                                            │ +│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced command groups ────────────────────────────────────────────────────────────────────────────────────────────╮ +│testing                Tools that developers can use to run tests                                                 │ +│ci-image               Tools that developers can use to manually manage CI images                                 │ +│k8s                    Tools that developers use to run Kubernetes tests                                          │ +│prod-image             Tools that developers can use to manually manage PROD images                               │ +│setup                  Tools that developers can use to configure Breeze                                          │ +│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ +│ci                     Tools that CI workflows use to cleanup/manage CI environment                               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_ci-image_build.svg b/images/breeze/output_ci-image_build.svg index 8319e04d12496..05742c982012d 100644 --- a/images/breeze/output_ci-image_build.svg +++ b/images/breeze/output_ci-image_build.svg @@ -280,7 +280,7 @@ ╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ │--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ │[default: 3.7]                                              │ │--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ │--upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when regular build      │ @@ -300,7 +300,7 @@ │(INTEGER RANGE)                                                            │ │[default: 4; 1<=x<=8]                                                      │ │--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ +│[default: 3.7 3.8 3.9 3.10 3.11]                                              │ │--skip-cleanupSkip cleanup of temporary files created during parallel run.│ │--debug-resourcesWhether to show resource information while running in parallel.│ │--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ diff --git a/images/breeze/output_ci-image_pull.svg b/images/breeze/output_ci-image_pull.svg index d0e5c8d141651..3c54da94e3bb4 100644 --- a/images/breeze/output_ci-image_pull.svg +++ b/images/breeze/output_ci-image_pull.svg @@ -1,4 +1,4 @@ - + ╭─ Pull image flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ │--image-tag-tTag of the image which is used to pull the image.(TEXT)[default: latest]│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -│--verifyVerify image.│ -│--wait-for-imageWait until image is available.│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful when you   │ -│build or pull image with --image-tag.                                                      │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ +│[default: 3.7]                                              │ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--verifyVerify image.│ +│--wait-for-imageWait until image is available.│ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful when you   │ +│build or pull image with --image-tag.                                                      │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10 3.11]                                              │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_ci-image_verify.svg b/images/breeze/output_ci-image_verify.svg index 6c18349eed7f2..d32439aa4db70 100644 --- a/images/breeze/output_ci-image_verify.svg +++ b/images/breeze/output_ci-image_verify.svg @@ -1,4 +1,4 @@ - +