From 2968ac8e65ce6c5bef5ee228e524b3d75eb3dae3 Mon Sep 17 00:00:00 2001 From: Puckel_ Date: Wed, 15 Jan 2020 10:47:16 +0100 Subject: [PATCH] Bump to Airflow 1.10.7 --- Dockerfile | 11 +++++++---- README.md | 4 ++-- config/airflow.cfg | 13 +++++++++++++ docker-compose-CeleryExecutor.yml | 8 ++++---- docker-compose-LocalExecutor.yml | 13 ++++++++++--- script/entrypoint.sh | 4 ++-- 6 files changed, 38 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index ba709a84..b1bba817 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# VERSION 1.10.6 +# VERSION 1.10.7 # AUTHOR: Matthieu "Puckel_" Roisil # DESCRIPTION: Basic Airflow container # BUILD: docker build --rm -t puckel/docker-airflow . @@ -7,12 +7,12 @@ FROM python:3.7-slim-stretch LABEL maintainer="Puckel_" -# Never prompts the user for choices on installation/configuration of packages +# Never prompt the user for choices on installation/configuration of packages ENV DEBIAN_FRONTEND noninteractive ENV TERM linux # Airflow -ARG AIRFLOW_VERSION=1.10.6 +ARG AIRFLOW_VERSION=1.10.7 ARG AIRFLOW_USER_HOME=/usr/local/airflow ARG AIRFLOW_DEPS="" ARG PYTHON_DEPS="" @@ -25,6 +25,9 @@ ENV LC_ALL en_US.UTF-8 ENV LC_CTYPE en_US.UTF-8 ENV LC_MESSAGES en_US.UTF-8 +# Disable noisy "Handling signal" log messages: +# ENV GUNICORN_CMD_ARGS --log-level WARNING + RUN set -ex \ && buildDeps=' \ freetds-dev \ @@ -80,4 +83,4 @@ EXPOSE 8080 5555 8793 USER airflow WORKDIR ${AIRFLOW_USER_HOME} ENTRYPOINT ["/entrypoint.sh"] -CMD ["webserver"] # set default arg for entrypoint +CMD ["webserver"] diff --git a/README.md b/README.md index 11e64b9e..f8fd6621 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ For encrypted connection passwords (in Local or Celery Executor), you must have docker run puckel/docker-airflow python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print(FERNET_KEY)" -## Configurating Airflow +## Configuring Airflow It's possible to set any configuration value for Airflow from environment variables, which are used over values from the airflow.cfg. @@ -124,4 +124,4 @@ You can also use this to run a bash shell or any other command in the same envir # Wanna help? -Fork, improve and PR. ;-) +Fork, improve and PR. diff --git a/config/airflow.cfg b/config/airflow.cfg index a3cda086..06f08947 100644 --- a/config/airflow.cfg +++ b/config/airflow.cfg @@ -174,6 +174,14 @@ dag_discovery_safe_mode = True # The number of retries each task is going to have by default. Can be overridden at dag or task level. default_task_retries = 0 +# Whether to serialises DAGs and persist them in DB. +# If set to True, Webserver reads from DB instead of parsing DAG files +# More details: https://airflow.apache.org/howto/enable-dag-serialization.html +store_serialized_dags = False + +# Updating serialized DAG can not be faster than a minimum interval to reduce database write rate. +min_serialized_dag_update_interval = 30 + [cli] # In what way should the cli access the API. The LocalClient will use the # database directly, while the json_client will use the api running on the @@ -682,6 +690,11 @@ env_from_secret_ref = git_repo = git_branch = git_subpath = + +# The specific rev or hash the git_sync init container will checkout +# This becomes GIT_SYNC_REV environment variable in the git_sync init container for worker pods +git_sync_rev = + # Use git_user and git_password for user authentication or git_ssh_key_secret_name and git_ssh_key_secret_key # for SSH authentication git_user = diff --git a/docker-compose-CeleryExecutor.yml b/docker-compose-CeleryExecutor.yml index 7d2a7ce7..8db540ee 100644 --- a/docker-compose-CeleryExecutor.yml +++ b/docker-compose-CeleryExecutor.yml @@ -16,7 +16,7 @@ services: # - ./pgdata:/var/lib/postgresql/data/pgdata webserver: - image: puckel/docker-airflow:1.10.6 + image: puckel/docker-airflow:1.10.7 restart: always depends_on: - postgres @@ -43,7 +43,7 @@ services: retries: 3 flower: - image: puckel/docker-airflow:1.10.6 + image: puckel/docker-airflow:1.10.7 restart: always depends_on: - redis @@ -55,7 +55,7 @@ services: command: flower scheduler: - image: puckel/docker-airflow:1.10.6 + image: puckel/docker-airflow:1.10.7 restart: always depends_on: - webserver @@ -74,7 +74,7 @@ services: command: scheduler worker: - image: puckel/docker-airflow:1.10.6 + image: puckel/docker-airflow:1.10.7 restart: always depends_on: - scheduler diff --git a/docker-compose-LocalExecutor.yml b/docker-compose-LocalExecutor.yml index 7098a149..7091a9be 100644 --- a/docker-compose-LocalExecutor.yml +++ b/docker-compose-LocalExecutor.yml @@ -1,4 +1,4 @@ -version: '2.1' +version: '3.7' services: postgres: image: postgres:9.6 @@ -6,18 +6,25 @@ services: - POSTGRES_USER=airflow - POSTGRES_PASSWORD=airflow - POSTGRES_DB=airflow + logging: + options: + max-size: 10m + max-file: "3" webserver: - image: puckel/docker-airflow:1.10.6 + image: puckel/docker-airflow:1.10.7 restart: always depends_on: - postgres environment: - LOAD_EX=n - EXECUTOR=Local + logging: + options: + max-size: 10m + max-file: "3" volumes: - ./dags:/usr/local/airflow/dags - # Uncomment to include custom plugins # - ./plugins:/usr/local/airflow/plugins ports: - "8080:8080" diff --git a/script/entrypoint.sh b/script/entrypoint.sh index fae9bb49..06532713 100755 --- a/script/entrypoint.sh +++ b/script/entrypoint.sh @@ -27,7 +27,7 @@ export \ AIRFLOW__CORE__SQL_ALCHEMY_CONN \ -# Load DAGs exemples (default: Yes) +# Load DAGs examples (default: Yes) if [[ -z "$AIRFLOW__CORE__LOAD_EXAMPLES" && "${LOAD_EX:=n}" == n ]] then AIRFLOW__CORE__LOAD_EXAMPLES=False @@ -79,7 +79,7 @@ case "$1" in exec airflow webserver ;; worker|scheduler) - # To give the webserver time to run initdb. + # Give the webserver time to run initdb. sleep 10 exec airflow "$@" ;;