Dataset Pull Automation #213
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Dataset Pull Automation | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| notifySlack: | |
| description: 'Notify Slack' | |
| required: true | |
| default: true | |
| type: boolean | |
| commitToRepos: | |
| description: 'Commit to opendata.scot' | |
| required: true | |
| default: true | |
| type: boolean | |
| schedule: | |
| - cron: '0 12 * * FRI' | |
| jobs: | |
| Pull_datasets: | |
| runs-on: ubuntu-latest | |
| env: | |
| jkanPath: jkan | |
| odbodsPath: the_od_bods | |
| steps: | |
| # Init repos | |
| - name: Check out OD bods toolset repo | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: 'OpenDataScotland/the_od_bods' | |
| path: 'the_od_bods' | |
| token: ${{ secrets.REPO_COMMIT_TOKEN }} | |
| - name: Check out JKAN repo | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: 'OpenDataScotland/jkan' | |
| path: 'jkan' | |
| token: ${{ secrets.REPO_COMMIT_TOKEN }} | |
| # Run pipeline | |
| - name: Build and run docker container | |
| run: | | |
| docker build -t open_data_scotland_pipeline . | |
| docker run -t --name ods_pipeline open_data_scotland_pipeline | |
| working-directory: ${{env.odbodsPath}} | |
| # Extract files from Docker container | |
| - name: Copy container contents and overwrite datasets | |
| run: | | |
| rm -r jkan/_datasets/* | |
| docker cp ods_pipeline:./usr/src/app/. . | |
| # Commit changes | |
| - name: Git config | |
| working-directory: ${{env.jkanPath}} | |
| run: | | |
| git config --global user.name 'github-actions[bot]' | |
| git config --global user.email 'github-actions[bot]@users.noreply.github.com' | |
| - name: Git status JKAN | |
| run: git status | |
| working-directory: ${{env.jkanPath}} | |
| - name: Commit and push JKAN | |
| if: ${{ github.event.inputs.commitToRepos == 'true' || github.event.schedule == '0 12 * * FRI' }} | |
| run: git add -A && git commit -m 'Dataset Sync' && git push origin gh-pages | |
| working-directory: ${{env.jkanPath}} | |
| - name: Git status the_od_bods | |
| run: git status | |
| working-directory: ${{env.odbodsPath}} | |
| - name: Commit and push the_od_bods | |
| if: ${{ github.event.inputs.commitToRepos == 'true' || github.event.schedule == '0 12 * * FRI' }} | |
| run: git add data && git commit -m "Dataset Sync" && git push origin main | |
| working-directory: ${{env.odbodsPath}} | |
| # Tell the Slack channel! | |
| - name: Slack Notification | |
| if: ${{ github.event.inputs.notifySlack == 'true' || github.event.schedule == '0 12 * * FRI' }} | |
| working-directory: ${{env.jkanPath}} | |
| env: | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} | |
| run: | | |
| COMMIT=$(git rev-parse HEAD) | |
| curl --location --request POST ''$SLACK_WEBHOOK'' \ | |
| --header 'Content-Type: application/json' \ | |
| --data-raw '{"text": "I just did a sync!\nCheck out the commit information here: https://github.com/OpenDataScotland/jkan/commit/'$COMMIT'", | |
| "icon_url":"https://opendata.scot/img/ods_logo.png", | |
| "username":"ODS bot" | |
| }' | |
| # Debugging: artifact upload if the workflow fails at all or if we're not committing to the site (likely to debug) | |
| - uses: actions/upload-artifact@v3 | |
| if: ${{ failure() || github.event.inputs.commitToRepos == 'false' }} | |
| with: | |
| name: workspace | |
| path: ${{ github.workspace }} | |
| # Log output | |
| - name: Log output | |
| run: cat log.md >> $GITHUB_STEP_SUMMARY | |
| working-directory: ${{env.odbodsPath}} | |