pipeline.yml

name: Dataset Pull Automation

on:
  workflow_dispatch:
    inputs:
      notifySlack:
        description: 'Notify Slack'
        required: true
        default: true
        type: boolean
      commitToRepos:
        description: 'Commit to opendata.scot'
        required: true
        default: true
        type: boolean

  schedule:
    - cron: '0 12 * * FRI'

jobs:
  Pull_datasets:
    runs-on: ubuntu-latest
    env: 
      jkanPath: jkan
      odbodsPath: the_od_bods
      
    steps:
    # Init repos
    - name: Check out OD bods toolset repo
      uses: actions/checkout@v3
      with: 
        repository: 'OpenDataScotland/the_od_bods'
        path: 'the_od_bods'
        token: ${{ secrets.REPO_COMMIT_TOKEN }}
    - name: Check out JKAN repo
      uses: actions/checkout@v3
      with: 
        repository: 'OpenDataScotland/jkan'
        path: 'jkan'
        token: ${{ secrets.REPO_COMMIT_TOKEN }}

    # Run pipeline
    - name: Build and run docker container
      run: |
        docker build -t open_data_scotland_pipeline .
        docker run -t --name ods_pipeline open_data_scotland_pipeline
      working-directory: ${{env.odbodsPath}}

    # Extract files from Docker container
    - name: Copy container contents and overwrite datasets
      run: |
        rm -r jkan/_datasets/*
        docker cp ods_pipeline:./usr/src/app/. .

    # Commit changes
    - name: Git config
      working-directory: ${{env.jkanPath}}
      run: |
        git config --global user.name 'github-actions[bot]'
        git config --global user.email 'github-actions[bot]@users.noreply.github.com'
    - name: Git status JKAN
      run: git status
      working-directory: ${{env.jkanPath}}
    - name: Commit and push JKAN
      if: ${{ github.event.inputs.commitToRepos == 'true' || github.event.schedule == '0 12 * * FRI' }} 
      run: git add -A && git commit -m 'Dataset Sync' && git push origin gh-pages
      working-directory: ${{env.jkanPath}}
    - name: Git status the_od_bods
      run: git status
      working-directory: ${{env.odbodsPath}}
    - name: Commit and push the_od_bods
      if: ${{ github.event.inputs.commitToRepos == 'true' || github.event.schedule == '0 12 * * FRI' }} 
      run: git add data && git commit -m "Dataset Sync" && git push origin main
      working-directory: ${{env.odbodsPath}}

    # Tell the Slack channel!
    - name: Slack Notification
      if: ${{ github.event.inputs.notifySlack == 'true' || github.event.schedule == '0 12 * * FRI' }} 
      working-directory: ${{env.jkanPath}}
      env:
        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
      run: | 
        COMMIT=$(git rev-parse HEAD)
        curl --location --request POST ''$SLACK_WEBHOOK'' \
        --header 'Content-Type: application/json' \
        --data-raw '{"text": "I just did a sync!\nCheck out the commit information here: https://github.com/OpenDataScotland/jkan/commit/'$COMMIT'",
        "icon_url":"https://opendata.scot/img/ods_logo.png",
        "username":"ODS bot"
        }'
        
    # Debugging: artifact upload if the workflow fails at all or if we're not committing to the site (likely to debug)
    - uses: actions/upload-artifact@v3
      if: ${{ failure() || github.event.inputs.commitToRepos == 'false' }}
      with:
        name: workspace
        path: ${{ github.workspace }}
        
    # Log output
    - name: Log output
      run: cat log.md >> $GITHUB_STEP_SUMMARY
      working-directory: ${{env.odbodsPath}}