From 67363fe38b18ee3e19f3c6d1f0c0eb3f2af3c906 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Mon, 30 Sep 2024 11:10:32 -0500 Subject: [PATCH 1/4] test bundle builds job before push updated bundles. --- .github/workflows/bundle_cron.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.github/workflows/bundle_cron.yml b/.github/workflows/bundle_cron.yml index 43a0a2e..a47999f 100644 --- a/.github/workflows/bundle_cron.yml +++ b/.github/workflows/bundle_cron.yml @@ -20,8 +20,37 @@ jobs: run: | echo "This workflow will only run if Adafruit is the repository owner." echo "Repository owner is Adafruit: $OWNER_IS_ADAFRUIT" + test-bundle-builds: + runs-on: ubuntu-latest + steps: + - name: Set up Python 3.12 + uses: actions/setup-python@v4 + with: + python-version: 3.12 + - name: Versions + run: | + python3 --version + - uses: actions/checkout@v3 + with: + submodules: true + - name: Install deps + run: | + pip install -r requirements.txt + - name: Test Building Community Bundle + run: | + git clone --recurse-submodules https://github.com/adafruit/CircuitPython_Community_Bundle.git + cd CircuitPython_Community_Bundle + circuitpython-build-bundles --filename_prefix test-community-bundle --library_location libraries --library_depth 2 + - name: Test Building Adafruit Bundle + run: | + git clone --recurse-submodules https://github.com/adafruit/Adafruit_CircuitPython_Bundle.git + cd Adafruit_CircuitPython_Bundle + circuitpython-build-bundles --filename_prefix test-adafruit-bundle --library_location libraries --library_depth 2 + update-bundles: runs-on: ubuntu-latest + # Only run if test-bundle-builds succeeds + needs: test-bundle-builds # Only run the build on Adafruit's repository. Forks won't have the secrets. # Its necessary to do this here, since 'schedule' events cannot (currently) # be limited (they run on all forks' default branches). From 1d5fde846b9781b84e47da8b91d1bcb3bd89447c Mon Sep 17 00:00:00 2001 From: foamyguy Date: Mon, 30 Sep 2024 11:21:26 -0500 Subject: [PATCH 2/4] add build-tools to reqs --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 2d24158..dc0c477 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ # SPDX-License-Identifier: MIT black==24.3.0 +circuitpython-build-tools packaging==22.0 pylint==3.2.6 pytest From 1b1666589459299810d10cc1916ec8ab75b90650 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Sat, 5 Oct 2024 11:58:00 -0500 Subject: [PATCH 3/4] revert bundle_cron to match main. update circuitpython_bundle with changes from main. add test build bundle logic. remove unneeded if statement. allow RuntimeError to be raised. --- .github/workflows/bundle_cron.yml | 46 ++++------------------- adabot/circuitpython_bundle.py | 61 +++++++++++++++++++++++-------- 2 files changed, 53 insertions(+), 54 deletions(-) diff --git a/.github/workflows/bundle_cron.yml b/.github/workflows/bundle_cron.yml index a47999f..2acc3b0 100644 --- a/.github/workflows/bundle_cron.yml +++ b/.github/workflows/bundle_cron.yml @@ -7,6 +7,7 @@ name: Update Bundles on: schedule: - cron: 0 5 * * * + workflow_dispatch: jobs: check-repo-owner: @@ -20,52 +21,22 @@ jobs: run: | echo "This workflow will only run if Adafruit is the repository owner." echo "Repository owner is Adafruit: $OWNER_IS_ADAFRUIT" - test-bundle-builds: - runs-on: ubuntu-latest - steps: - - name: Set up Python 3.12 - uses: actions/setup-python@v4 - with: - python-version: 3.12 - - name: Versions - run: | - python3 --version - - uses: actions/checkout@v3 - with: - submodules: true - - name: Install deps - run: | - pip install -r requirements.txt - - name: Test Building Community Bundle - run: | - git clone --recurse-submodules https://github.com/adafruit/CircuitPython_Community_Bundle.git - cd CircuitPython_Community_Bundle - circuitpython-build-bundles --filename_prefix test-community-bundle --library_location libraries --library_depth 2 - - name: Test Building Adafruit Bundle - run: | - git clone --recurse-submodules https://github.com/adafruit/Adafruit_CircuitPython_Bundle.git - cd Adafruit_CircuitPython_Bundle - circuitpython-build-bundles --filename_prefix test-adafruit-bundle --library_location libraries --library_depth 2 - update-bundles: runs-on: ubuntu-latest - # Only run if test-bundle-builds succeeds - needs: test-bundle-builds # Only run the build on Adafruit's repository. Forks won't have the secrets. # Its necessary to do this here, since 'schedule' events cannot (currently) # be limited (they run on all forks' default branches). if: startswith(github.repository, 'adafruit/') - services: - redis: - image: redis - ports: - - 6379/tcp - options: --entrypoint redis-server steps: - - name: Set up Python 3.9 + - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: 3.12 + - name: Load contributor cache + uses: actions/cache@v4 + with: + key: "contributor-cache" + path: "contributors.json" - name: Versions run: | python3 --version @@ -80,7 +51,6 @@ jobs: ADABOT_EMAIL: ${{ secrets.ADABOT_EMAIL }} ADABOT_GITHUB_USER: ${{ secrets.ADABOT_GITHUB_USER }} ADABOT_GITHUB_ACCESS_TOKEN: ${{ secrets.ADABOT_GITHUB_ACCESS_TOKEN }} - REDIS_PORT: ${{ job.services.redis.ports[6379] }} BIGQUERY_PRIVATE_KEY: ${{ secrets.BIGQUERY_PRIVATE_KEY }} BIGQUERY_CLIENT_EMAIL: ${{ secrets.BIGQUERY_CLIENT_EMAIL }} run: | diff --git a/adabot/circuitpython_bundle.py b/adabot/circuitpython_bundle.py index b1789d2..90c5a87 100644 --- a/adabot/circuitpython_bundle.py +++ b/adabot/circuitpython_bundle.py @@ -6,30 +6,28 @@ If updates are found the bundle is updated, updates are pushed to the remote, and a new release is made. """ - +import contextlib from datetime import date from io import StringIO +import json import os +import pathlib import shlex import subprocess -import redis as redis_py - import sh +from circuitpython_build_tools.scripts.build_bundles import build_bundles from sh.contrib import git from adabot import github_requests as gh_reqs from adabot.lib import common_funcs from adabot import circuitpython_library_download_stats as dl_stats -REDIS = None -if "GITHUB_WORKSPACE" in os.environ: - REDIS = redis_py.StrictRedis(port=os.environ["REDIS_PORT"]) -else: - REDIS = redis_py.StrictRedis() BUNDLES = ["Adafruit_CircuitPython_Bundle", "CircuitPython_Community_Bundle"] +CONTRIBUTOR_CACHE = {} + def fetch_bundle(bundle, bundle_path): """Clones `bundle` to `bundle_path`""" @@ -380,20 +378,17 @@ def get_contributors(repo, commit_range): return contributors for log_line in output.split("\n"): sha, author_email, committer_email = log_line.split(",") - author = REDIS.get("github_username:" + author_email) - committer = REDIS.get("github_username:" + committer_email) + author = CONTRIBUTOR_CACHE.get("github_username:" + author_email, None) + committer = CONTRIBUTOR_CACHE.get("github_username:" + committer_email, None) if not author or not committer: github_commit_info = gh_reqs.get("/repos/" + repo + "/commits/" + sha) github_commit_info = github_commit_info.json() if github_commit_info["author"]: author = github_commit_info["author"]["login"] - REDIS.set("github_username:" + author_email, author) + CONTRIBUTOR_CACHE["github_username:" + author_email] = author if github_commit_info["committer"]: committer = github_commit_info["committer"]["login"] - REDIS.set("github_username:" + committer_email, committer) - else: - author = author.decode("utf-8") - committer = committer.decode("utf-8") + CONTRIBUTOR_CACHE["github_username:" + committer_email] = committer if committer_email == "noreply@github.com": committer = None @@ -425,6 +420,26 @@ def add_contributors(master_list, additions): master_list[contributor] += additions[contributor] +def test_bundle_build(bundle_dir): + """ + Attempts to build the bundle at the given location. + Returns exit code 0 if success. + Returns exit code >0 if failed to build. + """ + with contextlib.chdir(bundle_dir): + build_bundles( + [ + "--filename_prefix", + "test-build-bundle", + "--library_location", + "libraries", + "--library_depth", + "2", + ], + standalone_mode=False, + ) + + # pylint: disable=too-many-locals,too-many-branches,too-many-statements def new_release(bundle, bundle_path): """Creates a new release for `bundle`.""" @@ -506,7 +521,7 @@ def new_release(bundle, bundle_path): release_description.append( "The libraries in each release are compiled for all recent major versions of CircuitPython." " Please download the one that matches the major version of your CircuitPython. For example" - ", if you are running 8.2.6 you should download the `8.x` bundle.\n" + ", if you are running 9.1.1 you should download the `9.x` bundle.\n" ) release_description.append( @@ -554,6 +569,10 @@ def new_release(bundle, bundle_path): if __name__ == "__main__": + contributor_cache_fn = pathlib.Path("contributors.json").resolve() + if contributor_cache_fn.exists(): + CONTRIBUTOR_CACHE = json.loads(contributor_cache_fn.read_text()) + bundles_dir = os.path.abspath(".bundles") if "GITHUB_WORKSPACE" in os.environ: git.config("--global", "user.name", "adabot") @@ -563,6 +582,13 @@ def new_release(bundle, bundle_path): try: fetch_bundle(cp_bundle, bundle_dir) updates, release_required = update_bundle(bundle_dir) + + # test bundle build and stop if it does not succeed + try: + test_bundle_build(bundle_dir) + except SystemExit as e: + if e.code != 0: + raise RuntimeError("Test Build of Bundle Failed") from e if release_required: commit_updates(bundle_dir, updates) push_updates(bundle_dir) @@ -570,3 +596,6 @@ def new_release(bundle, bundle_path): except RuntimeError as e: print("Failed to update and release:", cp_bundle) print(e) + raise e + finally: + contributor_cache_fn.write_text(json.dumps(CONTRIBUTOR_CACHE)) From 0b7eccc8333ac75a164cac086b5c1067abfd20e1 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Tue, 8 Oct 2024 09:48:10 -0500 Subject: [PATCH 4/4] reword docstring --- adabot/circuitpython_bundle.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/adabot/circuitpython_bundle.py b/adabot/circuitpython_bundle.py index 90c5a87..50722f9 100644 --- a/adabot/circuitpython_bundle.py +++ b/adabot/circuitpython_bundle.py @@ -423,8 +423,8 @@ def add_contributors(master_list, additions): def test_bundle_build(bundle_dir): """ Attempts to build the bundle at the given location. - Returns exit code 0 if success. - Returns exit code >0 if failed to build. + Raises system exit status 0 if successful. + Raises system exit status >0 if failed to build. """ with contextlib.chdir(bundle_dir): build_bundles(