From f423735a563eadbfc7a669e32c4bdc8642da9625 Mon Sep 17 00:00:00 2001 From: Puckel_ Date: Tue, 5 Sep 2017 11:22:03 +0200 Subject: [PATCH] Bump to 1.8.2 --- Dockerfile | 4 +-- circle.yml | 2 +- dags/tuto.py | 55 +++++++++++++++++++++++++++++++ docker-compose-CeleryExecutor.yml | 24 +++++++------- docker-compose-LocalExecutor.yml | 8 ++--- 5 files changed, 74 insertions(+), 19 deletions(-) create mode 100644 dags/tuto.py diff --git a/Dockerfile b/Dockerfile index eb4fcb08..067bcdd6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ # BUILD: docker build --rm -t puckel/docker-airflow . # SOURCE: https://github.com/puckel/docker-airflow -FROM python:3.6-stretch +FROM python:3.6-slim MAINTAINER Puckel_ # Never prompts the user for choices on installation/configuration of packages @@ -12,7 +12,7 @@ ENV DEBIAN_FRONTEND noninteractive ENV TERM linux # Airflow -ARG AIRFLOW_VERSION=1.8.1 +ARG AIRFLOW_VERSION=1.8.2 ARG AIRFLOW_HOME=/usr/local/airflow # Define en_US. diff --git a/circle.yml b/circle.yml index e7805a52..5981fe23 100644 --- a/circle.yml +++ b/circle.yml @@ -12,4 +12,4 @@ test: pre: - sleep 5 override: - - docker run puckel/docker-airflow version |grep '1.8.1' + - docker run puckel/docker-airflow version |grep '1.8.2' diff --git a/dags/tuto.py b/dags/tuto.py new file mode 100644 index 00000000..40a08453 --- /dev/null +++ b/dags/tuto.py @@ -0,0 +1,55 @@ +""" +Code that goes along with the Airflow located at: +http://airflow.readthedocs.org/en/latest/tutorial.html +""" +from airflow import DAG +from airflow.operators.bash_operator import BashOperator +from datetime import datetime, timedelta + + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': datetime(2015, 6, 1), + 'email': ['airflow@airflow.com'], + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=5), + # 'queue': 'bash_queue', + # 'pool': 'backfill', + # 'priority_weight': 10, + # 'end_date': datetime(2016, 1, 1), +} + +dag = DAG( + 'tutorial', default_args=default_args, schedule_interval=timedelta(1)) + +# t1, t2 and t3 are examples of tasks created by instantiating operators +t1 = BashOperator( + task_id='print_date', + bash_command='date', + dag=dag) + +t2 = BashOperator( + task_id='sleep', + bash_command='sleep 5', + retries=3, + dag=dag) + +templated_command = """ + {% for i in range(5) %} + echo "{{ ds }}" + echo "{{ macros.ds_add(ds, 7)}}" + echo "{{ params.my_param }}" + {% endfor %} +""" + +t3 = BashOperator( + task_id='templated', + bash_command=templated_command, + params={'my_param': 'Parameter I passed in'}, + dag=dag) + +t2.set_upstream(t1) +t3.set_upstream(t1) diff --git a/docker-compose-CeleryExecutor.yml b/docker-compose-CeleryExecutor.yml index ef100f76..ce0a4956 100644 --- a/docker-compose-CeleryExecutor.yml +++ b/docker-compose-CeleryExecutor.yml @@ -11,26 +11,26 @@ services: - POSTGRES_DB=airflow webserver: - image: puckel/docker-airflow:1.8.1-1 + image: puckel/docker-airflow:1.8.2 restart: always depends_on: - postgres - redis environment: - - LOAD_EX=y + - LOAD_EX=n - FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= - EXECUTOR=Celery # - POSTGRES_USER=airflow # - POSTGRES_PASSWORD=airflow # - POSTGRES_DB=airflow - # volumes: - # - ~/docker-airflow/dags:/usr/local/airflow/dags + volumes: + - ./dags:/usr/local/airflow/dags ports: - "8080:8080" command: webserver flower: - image: puckel/docker-airflow:1.8.1-1 + image: puckel/docker-airflow:1.8.2 restart: always depends_on: - redis @@ -41,14 +41,14 @@ services: command: flower scheduler: - image: puckel/docker-airflow:1.8.1-1 + image: puckel/docker-airflow:1.8.2 restart: always depends_on: - webserver - # volumes: - # - ~/docker-airflow/dags:/usr/local/airflow/dags + volumes: + - ./dags:/usr/local/airflow/dags environment: - - LOAD_EX=y + - LOAD_EX=n - FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= - EXECUTOR=Celery # - POSTGRES_USER=airflow @@ -57,12 +57,12 @@ services: command: scheduler worker: - image: puckel/docker-airflow:1.8.1-1 + image: puckel/docker-airflow:1.8.2 restart: always depends_on: - scheduler - # volumes: - # - ~/docker-airflow/dags:/usr/local/airflow/dags + volumes: + - ./dags:/usr/local/airflow/dags environment: - FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= - EXECUTOR=Celery diff --git a/docker-compose-LocalExecutor.yml b/docker-compose-LocalExecutor.yml index 21352869..d4f620d0 100644 --- a/docker-compose-LocalExecutor.yml +++ b/docker-compose-LocalExecutor.yml @@ -8,15 +8,15 @@ services: - POSTGRES_DB=airflow webserver: - image: puckel/docker-airflow:1.8.1-1 + image: puckel/docker-airflow:1.8.2 restart: always depends_on: - postgres environment: - - LOAD_EX=y + - LOAD_EX=n - EXECUTOR=Local - # volumes: - # - ~/github/docker-airflow/dags:/usr/local/airflow/dags + volumes: + - ./dags:/usr/local/airflow/dags ports: - "8080:8080" command: webserver