From 0d3866876a47c1c74e6bc812d2ce27f135a7c375 Mon Sep 17 00:00:00 2001 From: Aaron Claydon Date: Thu, 19 Dec 2024 23:45:51 +0000 Subject: [PATCH] start --- airflow/dags/batch-data-import.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/airflow/dags/batch-data-import.py b/airflow/dags/batch-data-import.py index b711652..9274b6c 100644 --- a/airflow/dags/batch-data-import.py +++ b/airflow/dags/batch-data-import.py @@ -4,6 +4,7 @@ from kubernetes.client import models as k8s from airflow import DAG +from airflow.operators.dummy import DummyOperator from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import KubernetesPodOperator from airflow.utils.dates import days_ago from airflow.hooks.base_hook import BaseHook @@ -137,6 +138,9 @@ def generate_job(name : str, command : str, instance_size : str = "small", taskg max_active_runs=1, concurrency=2, ) as dag: + start = DummyOperator(task_id="start") + end = DummyOperator(task_id="end") + stop_linker = generate_job("stop-linker", [ "data-linker", "run", "--type", "stopsDDDDD" ]) stop_indexer = generate_job("stop-indexer", [ "indexer", "stopsDDDDD" ]) @@ -173,4 +177,4 @@ def generate_job(name : str, command : str, instance_size : str = "small", taskg except yaml.YAMLError as exc: print(exc) - taskgroups["small"] >> taskgroups["medium"] >> taskgroups["large"] >> stop_linker >> stop_indexer + start >> taskgroups["small"] >> taskgroups["medium"] >> taskgroups["large"] >> stop_linker >> stop_indexer >> end