melty.yaml

name: Meltano Pipeline

on:
  pull_request:
    types: [opened, reopened, labeled, synchronize]
  push:
    branches: [main]
  workflow_dispatch:
    inputs:
      environment:
        description: Meltano Environment
        required: true
        default: prod
      logConfig:
        description: 'Logging configuration file'
        required: true
        default: 'logging/ci.yaml'
        type: choice
        options:
          - logging/ci.yaml
          - logging/dev.yaml
  schedule:
    - cron: '0 12 * * *' # Once a day around 6am CST (12am UTC)

env:
  MELTANO_DATABASE_URI: ${{ secrets.MELTANO_DATABASE_URI }}
  DUCKDB_PATH: ${{ secrets.DUCKDB_PATH }}
  MELTANO_CLI_LOG_CONFIG: ${{ github.event.inputs.environment || 'logging/ci.yaml' }}
  MELTANO_ENVIRONMENT: ${{ github.event.inputs.environment || 'prod' }}
  MOTHERDUCK_TOKEN: ${{ secrets.MOTHERDUCK_TOKEN }}

jobs:
  meltano-run:
    if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'  || contains(github.event.pull_request.labels.*.name, 'Test/ELTP') }}
    name: "Meltano Job: ${{ matrix.job }}"
    runs-on: ubuntu-latest
    container:
      image: ghcr.io/meltano/meltano:latest-python3.11
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GHCR_IO_TOKEN }}
      env:
        TAP_GETPOCKET_CONSUMER_KEY: ${{ secrets.TAP_GETPOCKET_CONSUMER_KEY }}
        TAP_GETPOCKET_ACCESS_TOKEN: ${{ secrets.TAP_GETPOCKET_ACCESS_TOKEN }}
        TAP_GITHUB_AUTH_TOKEN: ${{ secrets.TAP_GITHUB_AUTH_TOKEN }}
        TAP_READTHEDOCS_TOKEN: ${{ secrets.TAP_READTHEDOCS_TOKEN }}
        TAP_STACKEXCHANGE_KEY: ${{ secrets.TAP_STACKEXCHANGE_KEY }}
        TARGET_DUCKDB_PATH: ${{ secrets.TARGET_DUCKDB_PATH }}
    strategy:
      matrix:
        include:
          - job: "pocket-to-duckdb"
            schedule: "daily_pocket"
          - job: "stackoverflow-to-duckdb"
            schedule: "daily_stackoverflow"
          - job: "rtd-to-duckdb"
            schedule: "daily_readthedocs"
          - job: "gh-to-duckdb"
            schedule: "daily_github"
      fail-fast: false

    steps:
      - name: Checkout code
        uses: actions/checkout@v4.1.1

      - name: Check Meltano
        run: |
          meltano --version

      - name: Install Plugins
        run: |
          meltano install --schedule ${{ matrix.schedule }}

      - name: Run
        env:
          MELTANO_FF_STRICT_ENV_VAR_MODE: true
        run: |
          meltano run ${{ matrix.job }}

      - name: Rename metrics log
        run: |
          mv logs/singer_metrics.log logs/singer_metrics_${{ matrix.job }}.log

      - name: Upload metrics log
        uses: actions/upload-artifact@v3.1.3
        with:
          name: singer-metrics
          path: logs/singer_metrics_${{ matrix.job }}.log

  lint:
    name: Lint models
    runs-on: ubuntu-latest
    continue-on-error: true
    container:
      image: ghcr.io/meltano/meltano:latest-python3.11
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GHCR_IO_TOKEN }}
      env:
        DUCKDB_PATH: ${{ secrets.DUCKDB_PATH }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v4.1.1

      - name: Check Meltano
        run: |
          meltano --version

      - name: Install SQLFluff
        run: |
          meltano install utility sqlfluff

      - name: Lint
        env:
          MELTANO_CLI_LOG_LEVEL: info
        run: |
          meltano invoke sqlfluff:lint --format github-annotation-native -v transform/models evidence/sources

  transform:
    name: Data Transformation with dbt-duckdb
    runs-on: ubuntu-latest
    needs: [meltano-run, lint]
    container:
      image: ghcr.io/meltano/meltano:latest-python3.11
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GHCR_IO_TOKEN }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v4.1.1

      - name: Check Meltano
        run: |
          meltano --version

      - name: Install dbt-duckdb
        run: |
          meltano install transformer dbt-duckdb

      - name: Transform and Test
        run: |
          meltano run dbt-duckdb:build

  build-evidence:
    name: Build Evidence site locally
    needs: [transform]
    runs-on: ubuntu-latest
    container:
      image: ghcr.io/meltano/meltano:latest-python3.11
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GHCR_IO_TOKEN }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4.1.1

      - name: Check Meltano
        run: |
          meltano --version

      - name: Install evidence
        run: |
          meltano install transform dbt-duckdb
          meltano install utility evidence

      - name: Build
        run: |
          meltano invoke evidence build --strict

      - name: Upload Evidence build
        uses: actions/upload-artifact@v3.1.3
        with:
          name: evidence-build
          path: evidence/build

  deploy-evidence:
    name: Deploy Evidence site
    needs: [build-evidence]
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write
    steps:
      - name: Checkout code
        uses: actions/checkout@v4.1.1

      - name: Set Git config
        run: |
          git config --global --add safe.directory /__w/meltano-dataops/meltano-dataops

      - name: Download Evidence build
        uses: actions/download-artifact@v3.0.2
        with:
          name: evidence-build
          path: evidence/build

      - name: Deploy
        uses: amondnet/vercel-action@v25.1.1
        with:
          github-comment: ${{ github.event_name == 'pull_request' && 'true' || 'false' }}
          github-token: ${{ secrets.GITHUB_TOKEN }}
          vercel-token: ${{ secrets.VERCEL_TOKEN }}
          vercel-org-id: ${{ secrets.VERCEL_ORG_ID }}
          vercel-project-id: ${{ secrets.VERCEL_PROJECT_ID }}
          vercel-args: ${{ github.event_name == 'pull_request' && '' || '--prod' }}