chore: Update Meltano Plugins #1059
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Meltano Pipeline | |
| on: | |
| pull_request: | |
| types: [opened, reopened, labeled, synchronize] | |
| push: | |
| branches: [main] | |
| workflow_dispatch: | |
| inputs: | |
| environment: | |
| description: Meltano Environment | |
| required: true | |
| default: prod | |
| logConfig: | |
| description: 'Logging configuration file' | |
| required: true | |
| default: 'logging/ci.yaml' | |
| type: choice | |
| options: | |
| - logging/ci.yaml | |
| - logging/dev.yaml | |
| schedule: | |
| - cron: '0 12 * * *' # Once a day around 6am CST (12am UTC) | |
| env: | |
| MELTANO_DATABASE_URI: ${{ secrets.MELTANO_DATABASE_URI }} | |
| DUCKDB_PATH: ${{ secrets.DUCKDB_PATH }} | |
| MELTANO_CLI_LOG_CONFIG: ${{ github.event.inputs.environment || 'logging/ci.yaml' }} | |
| MELTANO_ENVIRONMENT: ${{ github.event.inputs.environment || 'prod' }} | |
| MOTHERDUCK_TOKEN: ${{ secrets.MOTHERDUCK_TOKEN }} | |
| jobs: | |
| meltano-run: | |
| if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'Test/ELTP') }} | |
| name: "Meltano Job: ${{ matrix.job }}" | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/meltano/meltano:latest-python3.11 | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GHCR_IO_TOKEN }} | |
| env: | |
| TAP_GETPOCKET_CONSUMER_KEY: ${{ secrets.TAP_GETPOCKET_CONSUMER_KEY }} | |
| TAP_GETPOCKET_ACCESS_TOKEN: ${{ secrets.TAP_GETPOCKET_ACCESS_TOKEN }} | |
| TAP_GITHUB_AUTH_TOKEN: ${{ secrets.TAP_GITHUB_AUTH_TOKEN }} | |
| TAP_READTHEDOCS_TOKEN: ${{ secrets.TAP_READTHEDOCS_TOKEN }} | |
| TAP_STACKEXCHANGE_KEY: ${{ secrets.TAP_STACKEXCHANGE_KEY }} | |
| TARGET_DUCKDB_PATH: ${{ secrets.TARGET_DUCKDB_PATH }} | |
| strategy: | |
| matrix: | |
| include: | |
| - job: "pocket-to-duckdb" | |
| schedule: "daily_pocket" | |
| - job: "stackoverflow-to-duckdb" | |
| schedule: "daily_stackoverflow" | |
| - job: "rtd-to-duckdb" | |
| schedule: "daily_readthedocs" | |
| - job: "gh-to-duckdb" | |
| schedule: "daily_github" | |
| fail-fast: false | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4.1.1 | |
| - name: Check Meltano | |
| run: | | |
| meltano --version | |
| - name: Install Plugins | |
| run: | | |
| meltano install --schedule ${{ matrix.schedule }} | |
| - name: Run | |
| env: | |
| MELTANO_FF_STRICT_ENV_VAR_MODE: true | |
| run: | | |
| meltano run ${{ matrix.job }} | |
| - name: Rename metrics log | |
| run: | | |
| mv logs/singer_metrics.log logs/singer_metrics_${{ matrix.job }}.log | |
| - name: Upload metrics log | |
| uses: actions/upload-artifact@v3.1.3 | |
| with: | |
| name: singer-metrics | |
| path: logs/singer_metrics_${{ matrix.job }}.log | |
| lint: | |
| name: Lint models | |
| runs-on: ubuntu-latest | |
| continue-on-error: true | |
| container: | |
| image: ghcr.io/meltano/meltano:latest-python3.11 | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GHCR_IO_TOKEN }} | |
| env: | |
| DUCKDB_PATH: ${{ secrets.DUCKDB_PATH }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4.1.1 | |
| - name: Check Meltano | |
| run: | | |
| meltano --version | |
| - name: Install SQLFluff | |
| run: | | |
| meltano install utility sqlfluff | |
| - name: Lint | |
| env: | |
| MELTANO_CLI_LOG_LEVEL: info | |
| run: | | |
| meltano invoke sqlfluff:lint --format github-annotation-native -v transform/models evidence/sources | |
| transform: | |
| name: Data Transformation with dbt-duckdb | |
| runs-on: ubuntu-latest | |
| needs: [meltano-run, lint] | |
| container: | |
| image: ghcr.io/meltano/meltano:latest-python3.11 | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GHCR_IO_TOKEN }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4.1.1 | |
| - name: Check Meltano | |
| run: | | |
| meltano --version | |
| - name: Install dbt-duckdb | |
| run: | | |
| meltano install transformer dbt-duckdb | |
| - name: Transform and Test | |
| run: | | |
| meltano run dbt-duckdb:build | |
| build-evidence: | |
| name: Build Evidence site locally | |
| needs: [transform] | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/meltano/meltano:latest-python3.11 | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GHCR_IO_TOKEN }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4.1.1 | |
| - name: Check Meltano | |
| run: | | |
| meltano --version | |
| - name: Install evidence | |
| run: | | |
| meltano install transform dbt-duckdb | |
| meltano install utility evidence | |
| - name: Build | |
| run: | | |
| meltano invoke evidence build --strict | |
| - name: Upload Evidence build | |
| uses: actions/upload-artifact@v3.1.3 | |
| with: | |
| name: evidence-build | |
| path: evidence/build | |
| deploy-evidence: | |
| name: Deploy Evidence site | |
| needs: [build-evidence] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4.1.1 | |
| - name: Set Git config | |
| run: | | |
| git config --global --add safe.directory /__w/meltano-dataops/meltano-dataops | |
| - name: Download Evidence build | |
| uses: actions/download-artifact@v3.0.2 | |
| with: | |
| name: evidence-build | |
| path: evidence/build | |
| - name: Deploy | |
| uses: amondnet/vercel-action@v25.1.1 | |
| with: | |
| github-comment: ${{ github.event_name == 'pull_request' && 'true' || 'false' }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| vercel-token: ${{ secrets.VERCEL_TOKEN }} | |
| vercel-org-id: ${{ secrets.VERCEL_ORG_ID }} | |
| vercel-project-id: ${{ secrets.VERCEL_PROJECT_ID }} | |
| vercel-args: ${{ github.event_name == 'pull_request' && '' || '--prod' }} |