elta_dev.yml

name: Extract, Load, Transform, and Analytics (Dev)
# TODO - add analytics

on:
  pull_request:
    branches:
      - main
    paths:
      # TODO - some jobs can be triggered even if no related paths were changed
      # There is a workaround solution applied in:
      #   https://github.com/gooddata/gdc-nas/blob/master/.github/workflows/pre-merge-pipeline.yml
      # Shared
      - .github/workflows/elta_dev.yml
      - .github/variables/elta_shared.yml
      # Extract load
      - data_pipeline/meltano.yml
      - data_pipeline/meltano_conf/**/*
      - data_pipeline/requirements-meltano.txt
      - .github/workflows/reusable_extract_load.yml
      # Transform
      - data_pipeline/macros/**/*
      - data_pipeline/models/**/*
      - data_pipeline/profile/**/*
      - data_pipeline/dbt_project.yml
      - data_pipeline/packages.yml
      - data_pipeline/requirements-dbt.txt
      - data_pipeline/requirements-gooddata.txt
      - .github/workflows/reusable_transform.yml

jobs:
  build-and-push-custom-meltano-image:
    uses: ./.github/workflows/reusable_build.yml
    with:
      TAGS: |
        ghcr.io/${{ github.repository }}/${{ vars.MELTANO_CUSTOM_IMAGE_BASE }}:${{ vars.MELTANO_VERSION }}
      DOCKER_FILE: Dockerfile_meltano
      BUILD_ARGS: |
        MELTANO_VERSION=${{ vars.MELTANO_VERSION }}
        IMAGES_WORKDIR=${{ vars.IMAGES_WORKDIR }}
    secrets: inherit
  extract-load-dev:
    needs: build-and-push-custom-meltano-image
    uses: ./.github/workflows/reusable_extract_load.yml
    with:
      INPUT_SCHEMA_FAA: "${{ vars.INPUT_SCHEMA_FAA }}"
      INPUT_SCHEMA_GITHUB: "${{ vars.INPUT_SCHEMA_GITHUB }}"
      INPUT_SCHEMA_EXCHANGERATEHOST: "${{ vars.INPUT_SCHEMA_EXCHANGERATEHOST }}"
      INPUT_SCHEMA_ECOMMERCE_DEMO: "${{ vars.INPUT_SCHEMA_ECOMMERCE_DEMO }}"
      INPUT_SCHEMA_DATA_SCIENCE: "${{ vars.INPUT_SCHEMA_DATA_SCIENCE }}"
      ENVIRONMENT: "dev"
      FULL_REFRESH: false # TODO: define workflow_dispatch with this parameter and set it here
      MELTANO_CUSTOM_IMAGE: "ghcr.io/${{ github.repository }}/${{ vars.MELTANO_CUSTOM_IMAGE_BASE }}:${{ vars.MELTANO_VERSION }}"
    secrets: inherit
#  transform-dev:
#    needs: extract-load-dev
#    uses: ./.github/workflows/reusable_transform.yml
#    with:
#      INPUT_SCHEMA_FAA: "${{ vars.INPUT_SCHEMA_FAA }}"
#      INPUT_SCHEMA_GITHUB: "${{ vars.INPUT_SCHEMA_GITHUB }}"
#      INPUT_SCHEMA_EXCHANGERATEHOST: "${{ vars.INPUT_SCHEMA_EXCHANGERATEHOST }}"
#      INPUT_SCHEMA_ECOMMERCE_DEMO: "${{ vars.INPUT_SCHEMA_ECOMMERCE_DEMO }}"
#      INPUT_SCHEMA_DATA_SCIENCE: "${{ vars.INPUT_SCHEMA_DATA_SCIENCE }}"
#      ENVIRONMENT: "dev"
#    secrets: inherit