elta_dev.yml

name: Extract, Load, Transform, and Analytics (Dev)

on:
  pull_request:
    branches:
      - main
    paths:
      # TODO - some jobs can be triggered even if no related paths were changed
      # There is a workaround solution applied in:
      #   https://github.com/gooddata/gdc-nas/blob/master/.github/workflows/pre-merge-pipeline.yml
      # Shared
      - .github/workflows/elta_dev.yml
      - .github/workflows/reusable_elta.yml
      - .env*
      - Makefile
      # Build custom Meltano image
      - data_pipeline/requirements-meltano.txt
      - data_pipeline/meltano-plugins.yml
      - data_pipeline/plugins/**/*.lock
      - Dockerfile_meltano
      - Dockerfile_dbt
      - Dockerfile_gooddata
      # Extract load
      - data_pipeline/meltano.yml
      - data_pipeline/meltano_conf/**/*
      - data_pipeline/requirements-meltano.txt
      - .github/workflows/reusable_extract_load.yml
      # Transform
      - data_pipeline/macros/**/*
      - data_pipeline/models/**/*
      - data_pipeline/profile/**/*
      - data_pipeline/dbt_project.yml
      - data_pipeline/packages.yml
      - data_pipeline/requirements-dbt.txt
      - data_pipeline/requirements-gooddata.txt
      - .github/workflows/reusable_transform.yml
      # Analytics
      - data_pipeline/gooddata_layouts/**/*
      - data_pipeline/requirements-gooddata.txt
      - .github/workflows/reusable_analytics.yml

jobs:
  read-env:
    runs-on: ubuntu-latest
    # Map a step output to a job output
    outputs:
      MELTANO_VERSION: ${{ steps.vars.outputs.MELTANO_VERSION }}
      DBT_VERSION: ${{ steps.vars.outputs.DBT_VERSION }}
      GOODDATA_VERSION: ${{ steps.vars.outputs.GOODDATA_VERSION }}
      PYTHON_IMAGE: ${{ steps.vars.outputs.PYTHON_IMAGE }}
      IMAGES_WORKDIR: ${{ steps.vars.outputs.IMAGES_WORKDIR }}
      MELTANO_CUSTOM_IMAGE: ${{ steps.vars.outputs.MELTANO_CUSTOM_IMAGE }}
      DBT_CUSTOM_IMAGE: ${{ steps.vars.outputs.DBT_CUSTOM_IMAGE }}
      GOODDATA_CUSTOM_IMAGE: ${{ steps.vars.outputs.GOODDATA_CUSTOM_IMAGE }}
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - id: vars
        shell: bash
        run: |
          source .env.dev
          echo "MELTANO_VERSION=$MELTANO_VERSION" >> "$GITHUB_OUTPUT"
          echo "DBT_VERSION=$DBT_VERSION" >> "$GITHUB_OUTPUT"
          echo "GOODDATA_VERSION=$GOODDATA_VERSION" >> "$GITHUB_OUTPUT"
          echo "PYTHON_IMAGE=$PYTHON_IMAGE" >> "$GITHUB_OUTPUT"
          echo "IMAGES_WORKDIR=$IMAGES_WORKDIR" >> "$GITHUB_OUTPUT"
          echo "MELTANO_CUSTOM_IMAGE=ghcr.io/${{ github.repository }}/$MELTANO_CUSTOM_IMAGE_BASE:$MELTANO_VERSION" >> "$GITHUB_OUTPUT"
          echo "DBT_CUSTOM_IMAGE=ghcr.io/${{ github.repository }}/$DBT_CUSTOM_IMAGE_BASE:$DBT_VERSION" >> "$GITHUB_OUTPUT"
          echo "GOODDATA_CUSTOM_IMAGE=ghcr.io/${{ github.repository }}/$GOODDATA_CLIENT_CUSTOM_IMAGE_BASE:$GOODDATA_VERSION" >> "$GITHUB_OUTPUT"

  build-images:
    needs:
      - read-env
    uses: ./.github/workflows/reusable_builds.yml
    with:
      MELTANO_CUSTOM_IMAGE: "${{ needs.read-env.outputs.MELTANO_CUSTOM_IMAGE }}"
      DBT_CUSTOM_IMAGE: "${{ needs.read-env.outputs.DBT_CUSTOM_IMAGE }}"
      GOODDATA_CUSTOM_IMAGE: "${{ needs.read-env.outputs.GOODDATA_CUSTOM_IMAGE }}"
      MELTANO_VERSION: ${{ needs.read-env.outputs.MELTANO_VERSION }}
      DBT_VERSION: ${{ needs.read-env.outputs.DBT_VERSION }}
      GOODDATA_VERSION: ${{ needs.read-env.outputs.GOODDATA_VERSION }}
      PYTHON_IMAGE: ${{ needs.read-env.outputs.PYTHON_IMAGE }}
      IMAGES_WORKDIR: "${{ needs.read-env.outputs.IMAGES_WORKDIR }}"
    secrets: inherit

  elta:
    needs:
      - build-images
    uses: ./.github/workflows/reusable_elta.yml
    with:
      ENVIRONMENT: "dev"
      MELTANO_CUSTOM_IMAGE: "${{ needs.read-env.outputs.MELTANO_CUSTOM_IMAGE }}"
      DBT_CUSTOM_IMAGE: "${{ needs.read-env.outputs.DBT_CUSTOM_IMAGE }}"
      GOODDATA_CUSTOM_IMAGE: "${{ needs.read-env.outputs.GOODDATA_CUSTOM_IMAGE }}"
    secrets: inherit

  elta-cloud:
    needs:
      - build-images
    uses: ./.github/workflows/reusable_elta.yml
    with:
      ENVIRONMENT: "dev_cloud"
      MELTANO_CUSTOM_IMAGE: "${{ needs.read-env.outputs.MELTANO_CUSTOM_IMAGE }}"
      DBT_CUSTOM_IMAGE: "${{ needs.read-env.outputs.DBT_CUSTOM_IMAGE }}"
      GOODDATA_CUSTOM_IMAGE: "${{ needs.read-env.outputs.GOODDATA_CUSTOM_IMAGE }}"
      DBT_CLOUD: "true"
    secrets: inherit

  # Branch for MotherDuck DB.
  # GoodData now supports MotherDuck only in Labs environment
  #   because the issue with extensions downloaded in runtime.
  elta-labs-motherduck:
    needs:
      - build-images
    uses: ./.github/workflows/reusable_elta.yml
    with:
      ENVIRONMENT: "dev"
      MELTANO_CUSTOM_IMAGE: "${{ needs.read-env.outputs.MELTANO_CUSTOM_IMAGE }}"
      DBT_CUSTOM_IMAGE: "${{ needs.read-env.outputs.DBT_CUSTOM_IMAGE }}"
      GOODDATA_CUSTOM_IMAGE: "${{ needs.read-env.outputs.GOODDATA_CUSTOM_IMAGE }}"
      DB_OVERRIDE: "motherduck"
    secrets: inherit