diff --git a/BREEZE.rst b/BREEZE.rst index b1a43d9f08fa1..1e3f9ea856edf 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -776,6 +776,12 @@ This is the current syntax for `./breeze <./breeze>`_: async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,mysql,postgres,redis,slack, ssh,statsd,virtualenv + --additional-extras + Additional extras to pass to build images The default is no additional extras. + + --additional-python-deps + Additional python dependencies to use when building the images. + -C, --force-clean-images Force build images with cache disabled. This will remove the pulled or build images and start building images from scratch. This might take a long time. @@ -1403,6 +1409,12 @@ This is the current syntax for `./breeze <./breeze>`_: async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,mysql,postgres,redis,slack, ssh,statsd,virtualenv + --additional-extras + Additional extras to pass to build images The default is no additional extras. + + --additional-python-deps + Additional python dependencies to use when building the images. + -C, --force-clean-images Force build images with cache disabled. This will remove the pulled or build images and start building images from scratch. This might take a long time. diff --git a/Dockerfile b/Dockerfile index dde22dccb6261..fc9c153446fa9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -168,10 +168,10 @@ ARG AIRFLOW_VERSION ENV AIRFLOW_VERSION=${AIRFLOW_VERSION} ARG AIRFLOW_EXTRAS -ARG ADDITIONAL_AIRFLOW_EXTRAS +ARG ADDITIONAL_AIRFLOW_EXTRAS="" ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}${ADDITIONAL_AIRFLOW_EXTRAS:+,}${ADDITIONAL_AIRFLOW_EXTRAS} -ARG ADDITIONAL_PYTHON_DEPS +ARG ADDITIONAL_PYTHON_DEPS="" ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS} ARG AIRFLOW_INSTALL_SOURCES="." diff --git a/Dockerfile.ci b/Dockerfile.ci index 6ab3c1fa876fd..1d040aefbfdb1 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -297,7 +297,8 @@ ENV AIRFLOW_BRANCH=${AIRFLOW_BRANCH} # Airflow Extras installed ARG AIRFLOW_EXTRAS="all" -ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS} +ARG ADDITIONAL_AIRFLOW_EXTRAS="" +ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}${ADDITIONAL_AIRFLOW_EXTRAS:+,}${ADDITIONAL_AIRFLOW_EXTRAS} RUN echo "Installing with extras: ${AIRFLOW_EXTRAS}." diff --git a/breeze b/breeze index 3674c7074cffc..db33a3a7b3d04 100755 --- a/breeze +++ b/breeze @@ -771,6 +771,14 @@ function parse_arguments() { export AIRFLOW_EXTRAS="${2}" echo "Extras : ${AIRFLOW_EXTRAS}" shift 2 ;; + --additional-extras) + export ADDITIONAL_AIRFLOW_EXTRAS="${2}" + echo "Additional extras : ${ADDITIONAL_AIRFLOW_EXTRAS}" + shift 2 ;; + --additional-python-deps) + export ADDITIONAL_PYTHON_DEPS="${2}" + echo "Additional python dependencies: ${ADDITIONAL_PYTHON_DEPS}" + shift 2 ;; -D|--dockerhub-user) export DOCKERHUB_USER="${2}" echo "Dockerhub user ${DOCKERHUB_USER}" @@ -1618,6 +1626,12 @@ ${FORMATTED_DEFAULT_CI_EXTRAS} Production image: ${FORMATTED_DEFAULT_PROD_EXTRAS} +--additional-extras + Additional extras to pass to build images The default is no additional extras. + +--additional-python-deps + Additional python dependencies to use when building the images. + -C, --force-clean-images Force build images with cache disabled. This will remove the pulled or build images and start building images from scratch. This might take a long time. diff --git a/breeze-complete b/breeze-complete index 1fa0d58fef6e3..0928810db227f 100644 --- a/breeze-complete +++ b/breeze-complete @@ -111,6 +111,7 @@ force-build-images force-pull-images production-image extras: force-clean-images dockerhub-user: dockerhub-repo: push-images postgres-version: mysql-version: version-suffix-for-pypi: version-suffix-for-svn: +additional-extras: additional-python-deps: " export BREEZE_COMMANDS=" diff --git a/scripts/ci/_utils.sh b/scripts/ci/_utils.sh index bc8c7fe2ada75..4b6061398a79d 100644 --- a/scripts/ci/_utils.sh +++ b/scripts/ci/_utils.sh @@ -1336,6 +1336,8 @@ Docker building ${AIRFLOW_CI_IMAGE}. --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg AIRFLOW_BRANCH="${BRANCH_NAME}" \ --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \ + --build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \ + --build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \ --build-arg AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD="${AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD}" \ --build-arg UPGRADE_TO_LATEST_REQUIREMENTS="${UPGRADE_TO_LATEST_REQUIREMENTS}" \ "${DOCKER_CACHE_CI_DIRECTIVE[@]}" \ @@ -1389,6 +1391,8 @@ function build_prod_image() { --build-arg PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \ + --build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \ + --build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \ "${DOCKER_CACHE_PROD_BUILD_DIRECTIVE[@]}" \ -t "${AIRFLOW_PROD_BUILD_IMAGE}" \ --target "airflow-build-image" \ @@ -1397,6 +1401,8 @@ function build_prod_image() { "${EXTRA_DOCKER_PROD_BUILD_FLAGS[@]}" \ --build-arg PYTHON_BASE_IMAGE="${PYTHON_BASE_IMAGE}" \ --build-arg PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}" \ + --build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \ + --build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \ "${DOCKER_CACHE_PROD_DIRECTIVE[@]}" \ @@ -1498,8 +1504,9 @@ function prepare_ci_build() { export IMAGE_DESCRIPTION="Airflow CI" export AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD="true" export AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS:="${DEFAULT_CI_EXTRAS}"}" + export ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS:=""}" + export ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS:=""}" export AIRFLOW_IMAGE="${AIRFLOW_CI_IMAGE}" - go_to_airflow_sources fix_group_permissions } @@ -1557,7 +1564,10 @@ function prepare_prod_build() { export THE_IMAGE_TYPE="PROD" export IMAGE_DESCRIPTION="Airflow production" export AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS:="${DEFAULT_PROD_EXTRAS}"}" + export ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS:=""}" + export ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS:=""}" export AIRFLOW_IMAGE="${AIRFLOW_PROD_IMAGE}" + if [[ ${CACHE_REGISTRY=""} != "" ]]; then echo "${CACHE_REGISTRY_PASSWORD}" | docker login \ --username "${CACHE_REGISTRY_USERNAME}" \ diff --git a/scripts/ci/in_container/kubernetes/docker/rebuild_airflow_image.sh b/scripts/ci/in_container/kubernetes/docker/rebuild_airflow_image.sh index 60ebffacc3395..f0306456198bc 100755 --- a/scripts/ci/in_container/kubernetes/docker/rebuild_airflow_image.sh +++ b/scripts/ci/in_container/kubernetes/docker/rebuild_airflow_image.sh @@ -44,6 +44,8 @@ docker build \ --build-arg PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \ + --build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \ + --build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \ --build-arg AIRFLOW_BRANCH="${AIRFLOW_BRANCH}" \ --build-arg AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD="${AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD}" \ --build-arg UPGRADE_TO_LATEST_REQUIREMENTS="${UPGRADE_TO_LATEST_REQUIREMENTS}" \