CD #7036
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CD | |
| on: | |
| push: | |
| branches: [ master ] | |
| schedule: | |
| # build once every 12 hours | |
| - cron: "0 */12 * * *" | |
| # allow manually trigger a build | |
| workflow_dispatch: | |
| jobs: | |
| CD: | |
| # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token | |
| # Number of commits to fetch. 0 indicates all history for all branches and tags. | |
| # Default: 1 | |
| # Only keep the latest 10 history to limit repo size | |
| fetch-depth: 0 # otherwise, you will failed to push refs to dest repo | |
| - name: Prepare cache | |
| run: | | |
| # Create empty cache files to allow `hashFiles` success. | |
| mkdir -p cache | |
| touch cache/nxdomains cache/okdomains cache/tbddomains cache/hash.txt cache/hash_old.txt | |
| # https://github.com/actions/cache | |
| - name: Cache domains status | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| cache/nxdomains | |
| cache/okdomains | |
| cache/tbddomains | |
| key: cache-domains-status-${{ hashFiles('cache/hash.txt') }} | |
| restore-keys: cache-domains-status-${{ hashFiles('cache/hash_old.txt') }} | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.11 | |
| architecture: x64 | |
| - name: Check build env | |
| run: | | |
| echo "pwd: $(pwd)" | |
| echo "USER: $USER" | |
| echo "HOME: $HOME" | |
| echo "SHELL: $SHELL" | |
| echo "PATH: $PATH" | |
| python3 --version | |
| pip3 --version | |
| mkdir -p cache | |
| ls -lh cache | |
| - name: Setup | |
| run: | | |
| # pip install binary to $HOME/.local/bin | |
| # export PATH="$PATH:$HOME/.local/bin" | |
| # | |
| # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#adding-a-system-path | |
| echo "/home/runner/.local/bin" >> $GITHUB_PATH | |
| make setup | |
| mkdir -p blocklists | |
| - name: Build blocklists from upstream | |
| run: | | |
| # make all | |
| # ./scripts/process_all.sh | |
| set -x | |
| # Remove wildcard domains from `cache/nxdomains` | |
| if [ -s cache/nxdomains ]; then | |
| # if cache/nxdomains is empty, grep return 1 will cause script failure. | |
| grep -v '*' cache/nxdomains > foo | |
| mv foo cache/nxdomains | |
| fi | |
| ./scripts/process.py ./sources/ad.json | |
| ./scripts/process.py ./sources/dating.json | |
| ./scripts/process.py ./sources/game.json | |
| ./scripts/process.py ./sources/gambling.json | |
| ./scripts/process.py ./sources/malicious.json | |
| ./scripts/process.py ./sources/piracy.json | |
| ./scripts/process.py ./sources/porn.json | |
| ./scripts/process.py ./sources/social_networks.json | |
| ./scripts/process.py ./sources/service.*.json | |
| wget https://openphish.com/feed.txt -O blocklists/phishing.txt | |
| # - run: ./scripts/process.py ./sources/ad.json | |
| # - run: ./scripts/process.py ./sources/dating.json | |
| # - run: ./scripts/process.py ./sources/gambling.json | |
| # - run: ./scripts/process.py ./sources/malicious.json | |
| # - run: ./scripts/process.py ./sources/piracy.json | |
| # - run: ./scripts/process.py ./sources/porn.json | |
| # - run: ./scripts/process.py ./sources/social_networks.json | |
| - name: Check changes and clean cache | |
| run: | | |
| ls -lh blocklists | |
| wc -l blocklists/*.domains.txt | sort -n | |
| ./scripts/clean_okcache.py | |
| ls -lh cache/nxdomains cache/okdomains cache/tbddomains cache/alldomains | |
| ./scripts/clean_cache.py | |
| ls -lh cache/nxdomains cache/okdomains cache/tbddomains cache/alldomains | |
| mv cache/hash.txt cache/hash_old.txt | |
| sha1sum cache/nxdomains cache/okdomains cache/tbddomains > cache/hash.txt | |
| git status | |
| - name: Git config | |
| run: | | |
| git config --local user.email "CD@github-action" | |
| git config --local user.name "GitHub Action" | |
| - name: Commit changes on release branch | |
| run: | | |
| git checkout --orphan release | |
| ./scripts/gen_readme.py > README.md | |
| wget --quiet https://github.com/jgm/pandoc/releases/download/3.0.1/pandoc-3.0.1-linux-amd64.tar.gz | |
| tar zxvf pandoc-3.0.1-linux-amd64.tar.gz | |
| chmod +x ./pandoc-3.0.1/bin/pandoc | |
| ./pandoc-3.0.1/bin/pandoc README.md -f markdown -t html -s --metadata title="dnss-blocklist" -o index.html | |
| git add sources blocklists icons README.md index.html 404.html | |
| git commit -m "Update blocklists from upstream" sources blocklists icons README.md index.html 404.html | |
| - name: Force push data branch changes | |
| uses: ad-m/github-push-action@master | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| # branch: ${{ github.ref }} | |
| branch: release | |
| force: true | |
| - name: Commit changes on master branch | |
| run: | | |
| git checkout master | |
| git add cache/hash.txt cache/hash_old.txt | |
| git commit -m "Update cache hash" cache/hash.txt cache/hash_old.txt | |
| git fetch | |
| git rebase | |
| - name: Push master branch changes | |
| uses: ad-m/github-push-action@master | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| branch: master | |
| force: false |