-
Notifications
You must be signed in to change notification settings - Fork 0
86 lines (74 loc) · 2.79 KB
/
scrape.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
# Heavily based on https://jacobian.org/til/github-actions-poetry/
name: Run Scraper
# Controls when the action will run. Workflow runs when manually triggered using the UI
# or API.
on:
schedule:
- cron: "40 15 * * *"
workflow_dispatch:
# Inputs the workflow accepts.
inputs:
source:
# Friendly description to be shown in the UI instead of 'name'
description: 'Source System'
# Default value if no value is explicitly provided
# Input has to be provided for the workflow to run
required: true
# The data type of the input
type: string
jobs:
scrape:
strategy:
matrix:
source:
[cfpb, doc, doj, dol, fec, ftc, hhs]
fail-fast: false
# The type of runner that the job will run on
runs-on: ubuntu-latest
environment: scraper
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.11'
# Cache the installation of Poetry itself, e.g. the next step. This prevents the workflow
# from installing Poetry every time, which can be slow. Note the use of the Poetry version
# number in the cache key, and the "-0" suffix: this allows you to invalidate the cache
# manually if/when you want to upgrade Poetry, or if something goes wrong. This could be
# mildly cleaner by using an environment variable, but I don't really care.
- name: cache poetry install
uses: actions/cache@v4
with:
path: ~/.local
key: poetry-1.6.1-0
- uses: snok/install-poetry@v1
with:
version: 1.6.1
virtualenvs-create: true
virtualenvs-in-project: true
- name: cache deps
id: cache-deps
uses: actions/cache@v4
with:
path: .venv
key: pydeps-${{ hashFiles('**/poetry.lock') }}
# Install dependencies. `--no-root` means "install all dependencies but not the project
# itself", which is what you want to avoid caching _your_ code. The `if` statement
# ensures this only runs on a cache miss.
- run: poetry install --no-interaction --no-root
if: steps.cache-deps.outputs.cache-hit != 'true'
- run: poetry install --no-interaction
- if: ${{ github.event_name == 'workflow_dispatch' }}
run: poetry run scrape ${{ inputs.source }}
env:
FEC_API_KEY: ${{ secrets.FEC_API_KEY }}
- if: ${{ github.event_name == 'schedule' }}
run: poetry run scrape ${{ matrix.source }}
env:
FEC_API_KEY: ${{ secrets.FEC_API_KEY }}
# publish the generated data files to github
- uses: mikeal/publish-to-github-action@master
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # GitHub sets this for you
BRANCH_NAME: 'main'