elta_dev.yml

name: Extract, Load, Transform, and Analytics (Dev)

on:
  pull_request:
    branches:
      - main
    paths:
      # TODO - some jobs can be triggered even if no related paths were changed
      # There is a workaround solution applied in:
      #   https://github.com/gooddata/gdc-nas/blob/master/.github/workflows/pre-merge-pipeline.yml
      # Shared
      - .github/workflows/elta_dev.yml
      - .github/workflows/reusable_elta.yml
      - .env.base
      - .env.prefix_dbt_vars
      - .env.dev*
      - Makefile
      # Build custom Meltano image
      - data_pipeline/requirements-meltano.txt
      - data_pipeline/meltano-plugins.yml
      - data_pipeline/plugins/**/*.lock
      - Dockerfile_meltano
      - Dockerfile_dbt
      - Dockerfile_gooddata
      # Extract load
      - data_pipeline/meltano.yml
      - data_pipeline/meltano_conf/**/*
      - data_pipeline/requirements-meltano.txt
      - .github/workflows/reusable_extract_load.yml
      # Transform
      - data_pipeline/macros/**/*
      - data_pipeline/models/**/*
      - data_pipeline/profile/**/*
      - data_pipeline/dbt_project.yml
      - data_pipeline/packages.yml
      - data_pipeline/requirements-dbt.txt
      - data_pipeline/requirements-gooddata.txt
      - .github/workflows/reusable_transform.yml
      # Analytics
      - data_pipeline/gooddata_layouts/**/*
      - data_pipeline/requirements-gooddata.txt
      - .github/workflows/reusable_analytics.yml

jobs:
  read-env:
    uses: ./.github/workflows/reusable_read_env.yml

  build-images:
    needs:
     - read-env
    uses: ./.github/workflows/reusable_builds.yml
    with:
      MELTANO_CUSTOM_IMAGE: "${{ needs.read-env.outputs.MELTANO_CUSTOM_IMAGE }}"
      DBT_CUSTOM_IMAGE: "${{ needs.read-env.outputs.DBT_CUSTOM_IMAGE }}"
      GOODDATA_SDK_CUSTOM_IMAGE: "${{ needs.read-env.outputs.GOODDATA_SDK_CUSTOM_IMAGE }}"
      MELTANO_VERSION: "${{ needs.read-env.outputs.MELTANO_VERSION }}"
      DBT_VERSION: "${{ needs.read-env.outputs.DBT_VERSION }}"
      GOODDATA_SDK_VERSION: "${{ needs.read-env.outputs.GOODDATA_SDK_VERSION }}"
      PYTHON_IMAGE: "${{ needs.read-env.outputs.PYTHON_IMAGE }}"
      IMAGES_WORKDIR: "${{ needs.read-env.outputs.IMAGES_WORKDIR }}"
    secrets: inherit

  elta:
    needs:
      - read-env
      - build-images
    uses: ./.github/workflows/reusable_elta.yml
    with:
      ENVIRONMENT: "dev"
      MELTANO_CUSTOM_IMAGE: "${{ needs.read-env.outputs.MELTANO_CUSTOM_IMAGE }}"
      DBT_CUSTOM_IMAGE: "${{ needs.read-env.outputs.DBT_CUSTOM_IMAGE }}"
      GOODDATA_SDK_CUSTOM_IMAGE: "${{ needs.read-env.outputs.GOODDATA_SDK_CUSTOM_IMAGE }}"
    secrets: inherit

  elta-cloud:
    needs:
      - read-env
      - build-images
    uses: ./.github/workflows/reusable_elta.yml
    with:
      ENVIRONMENT: "dev_cloud"
      MELTANO_CUSTOM_IMAGE: "${{ needs.read-env.outputs.MELTANO_CUSTOM_IMAGE }}"
      DBT_CUSTOM_IMAGE: "${{ needs.read-env.outputs.DBT_CUSTOM_IMAGE }}"
      GOODDATA_SDK_CUSTOM_IMAGE: "${{ needs.read-env.outputs.GOODDATA_SDK_CUSTOM_IMAGE }}"
      DBT_CLOUD: "true"
    secrets: inherit

  # Branch for MotherDuck DB.
  # GoodData now supports MotherDuck only in Labs environment
  #   because the issue with extensions downloaded in runtime.
  elta-labs-motherduck:
    needs:
      - read-env
      - build-images
    uses: ./.github/workflows/reusable_elta.yml
    with:
      ENVIRONMENT: "dev"
      MELTANO_CUSTOM_IMAGE: "${{ needs.read-env.outputs.MELTANO_CUSTOM_IMAGE }}"
      DBT_CUSTOM_IMAGE: "${{ needs.read-env.outputs.DBT_CUSTOM_IMAGE }}"
      GOODDATA_SDK_CUSTOM_IMAGE: "${{ needs.read-env.outputs.GOODDATA_SDK_CUSTOM_IMAGE }}"
      DB_OVERRIDE: "motherduck"
    secrets: inherit