diff --git a/.github/actions/deps/external/action.yml b/.github/actions/deps/external/action.yml index 5db7c04686a58..9e32e733a2ebc 100644 --- a/.github/actions/deps/external/action.yml +++ b/.github/actions/deps/external/action.yml @@ -9,54 +9,31 @@ inputs: - cache - restore - platform: + port: required: false default: none - type: choice - options: - - arm - - aarch - - esp - - riscv - - none + type: string runs: using: composite steps: - # aarch - - name: Get aarch toolchain - if: inputs.platform == 'aarch' - run: | - wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz - sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz - sudo apt-get install -y mtools - shell: bash - - name: Install mkfs.fat - if: inputs.platform == 'aarch' - run: | - wget https://github.com/dosfstools/dosfstools/releases/download/v4.2/dosfstools-4.2.tar.gz - tar -xaf dosfstools-4.2.tar.gz - cd dosfstools-4.2 - ./configure - make -j 2 - cd src - echo >> $GITHUB_PATH $(pwd) - shell: bash - # arm - name: Get arm toolchain - if: inputs.platform == 'aarch' || inputs.platform == 'arm' + if: >- + inputs.port != 'none' && + inputs.port != 'litex' && + inputs.port != 'espressif' uses: carlosperate/arm-none-eabi-gcc-action@v1 with: release: '10-2020-q4' - # esp - - name: Get esp toolchain - if: inputs.platform == 'esp' + # espressif + - name: Get espressif toolchain + if: inputs.port == 'espressif' run: sudo apt-get install -y ninja-build shell: bash - name: Install IDF tools - if: inputs.platform == 'esp' + if: inputs.port == 'espressif' run: | echo "Installing ESP-IDF tools" $IDF_PATH/tools/idf_tools.py --non-interactive install required @@ -66,24 +43,16 @@ runs: rm -rf $IDF_TOOLS_PATH/dist shell: bash - name: Set environment - if: inputs.platform == 'esp' + if: inputs.port == 'espressif' run: | source $IDF_PATH/export.sh echo >> $GITHUB_ENV "IDF_PYTHON_ENV_PATH=$IDF_PYTHON_ENV_PATH" echo >> $GITHUB_PATH "$PATH" shell: bash - # riscv - - name: Get riscv toolchain - if: inputs.platform == 'riscv' - run: | - wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz - sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz - shell: bash - # common - name: Cache python dependencies - if: inputs.platform != 'esp' + if: inputs.port != 'espressif' uses: ./.github/actions/deps/python with: action: ${{ inputs.action }} diff --git a/.github/actions/deps/ports/action.yml b/.github/actions/deps/ports/action.yml index a8c4f81a19770..8125de2acc487 100644 --- a/.github/actions/deps/ports/action.yml +++ b/.github/actions/deps/ports/action.yml @@ -19,10 +19,18 @@ runs: if [ -z $PORT ]; then (exit 1); else echo >> $GITHUB_OUTPUT "port=$PORT"; fi shell: bash - - name: Set up espressif port + - name: Set up broadcom + if: steps.board-to-port.outputs.port == 'broadcom' + uses: ./.github/actions/deps/ports/broadcom + + - name: Set up espressif if: steps.board-to-port.outputs.port == 'espressif' uses: ./.github/actions/deps/ports/espressif - - name: Set up nrf port + - name: Set up litex + if: steps.board-to-port.outputs.port == 'litex' + uses: ./.github/actions/deps/ports/litex + + - name: Set up nrf if: steps.board-to-port.outputs.port == 'nrf' uses: ./.github/actions/deps/ports/nrf diff --git a/.github/actions/deps/ports/broadcom/action.yml b/.github/actions/deps/ports/broadcom/action.yml new file mode 100644 index 0000000000000..bd76c78262922 --- /dev/null +++ b/.github/actions/deps/ports/broadcom/action.yml @@ -0,0 +1,21 @@ +name: Fetch broadcom port deps + +runs: + using: composite + steps: + - name: Get broadcom toolchain + run: | + wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz + sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz + sudo apt-get install -y mtools + shell: bash + - name: Install mkfs.fat + run: | + wget https://github.com/dosfstools/dosfstools/releases/download/v4.2/dosfstools-4.2.tar.gz + tar -xaf dosfstools-4.2.tar.gz + cd dosfstools-4.2 + ./configure + make -j 2 + cd src + echo >> $GITHUB_PATH $(pwd) + shell: bash diff --git a/.github/actions/deps/ports/litex/action.yml b/.github/actions/deps/ports/litex/action.yml new file mode 100644 index 0000000000000..a89a22250525e --- /dev/null +++ b/.github/actions/deps/ports/litex/action.yml @@ -0,0 +1,10 @@ +name: Fetch litex port deps + +runs: + using: composite + steps: + - name: Get litex toolchain + run: | + wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz + sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz + shell: bash diff --git a/.github/workflows/build-boards.yml b/.github/workflows/build-boards.yml index b975c131eec78..85e789bf87002 100644 --- a/.github/workflows/build-boards.yml +++ b/.github/workflows/build-boards.yml @@ -3,9 +3,6 @@ name: Build boards on: workflow_call: inputs: - platform: - required: true - type: string boards: required: true type: string @@ -19,7 +16,7 @@ on: required: false jobs: - build: + board: runs-on: ubuntu-22.04 env: CP_VERSION: ${{ inputs.cp-version }} @@ -38,6 +35,7 @@ jobs: with: python-version: 3.x - name: Set up port + id: set-up-port uses: ./.github/actions/deps/ports with: board: ${{ matrix.board }} @@ -47,7 +45,7 @@ jobs: - name: Set up external uses: ./.github/actions/deps/external with: - platform: ${{ inputs.platform }} + port: ${{ steps.set-up-port.outputs.port }} - name: Set up mpy-cross if: steps.set-up-submodules.outputs.frozen == 'True' uses: ./.github/actions/mpy_cross diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 19976f7c3d038..bee5afdd6aa8e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,16 +18,11 @@ concurrency: jobs: scheduler: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 outputs: - build-doc: ${{ steps.set-matrix.outputs.build-doc }} - build-boards: ${{ steps.set-matrix.outputs.build-boards }} - build-windows: ${{ steps.set-matrix.outputs.build-windows }} - boards-aarch: ${{ steps.set-matrix.outputs.boards-aarch }} - boards-arm: ${{ steps.set-matrix.outputs.boards-arm }} - boards-atmel: ${{ steps.set-matrix.outputs.boards-atmel }} - boards-esp: ${{ steps.set-matrix.outputs.boards-esp }} - boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }} + docs: ${{ steps.set-matrix.outputs.docs }} + ports: ${{ steps.set-matrix.outputs.ports }} + windows: ${{ steps.set-matrix.outputs.windows }} cp-version: ${{ steps.set-up-submodules.outputs.version }} steps: - name: Dump GitHub context @@ -99,7 +94,7 @@ jobs: mpy-cross: needs: scheduler - if: needs.scheduler.outputs.build-boards == 'True' + if: needs.scheduler.outputs.ports != '{}' uses: ./.github/workflows/build-mpy-cross.yml secrets: inherit with: @@ -108,7 +103,7 @@ jobs: mpy-cross-mac: runs-on: macos-11 needs: scheduler - if: needs.scheduler.outputs.build-boards == 'True' + if: needs.scheduler.outputs.ports != '{}' env: CP_VERSION: ${{ needs.scheduler.outputs.cp-version }} steps: @@ -160,10 +155,10 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - build-doc: + docs: runs-on: ubuntu-22.04 needs: scheduler - if: needs.scheduler.outputs.build-doc == 'True' + if: needs.scheduler.outputs.docs == 'True' env: CP_VERSION: ${{ needs.scheduler.outputs.cp-version }} steps: @@ -218,10 +213,10 @@ jobs: [ -z "$TWINE_USERNAME" ] || echo "Uploading dev release to PyPi" [ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/* - build-windows: + windows: runs-on: windows-2022 needs: scheduler - if: needs.scheduler.outputs.build-windows == 'True' + if: needs.scheduler.outputs.windows == 'True' env: CP_VERSION: ${{ needs.scheduler.outputs.cp-version }} defaults: @@ -286,52 +281,15 @@ jobs: # ERROR: Platform MINGW64_NT-10.0-17763-x86_64 appears to be unsupported # https://github.com/espressif/esp-idf/issues/7062 - aarch: - needs: [scheduler, mpy-cross, tests] - if: ${{ needs.scheduler.outputs.boards-aarch != '[]' }} - uses: ./.github/workflows/build-boards.yml - secrets: inherit - with: - platform: aarch - boards: ${{ needs.scheduler.outputs.boards-aarch }} - cp-version: ${{ needs.scheduler.outputs.cp-version }} - - arm: - needs: [scheduler, mpy-cross, tests] - if: ${{ needs.scheduler.outputs.boards-arm != '[]' }} - uses: ./.github/workflows/build-boards.yml - secrets: inherit - with: - platform: arm - boards: ${{ needs.scheduler.outputs.boards-arm }} - cp-version: ${{ needs.scheduler.outputs.cp-version }} - - atmel: - needs: [scheduler, mpy-cross, tests] - if: ${{ needs.scheduler.outputs.boards-atmel != '[]' }} - uses: ./.github/workflows/build-boards.yml - secrets: inherit - with: - platform: arm - boards: ${{ needs.scheduler.outputs.boards-atmel }} - cp-version: ${{ needs.scheduler.outputs.cp-version }} - - esp: - needs: [scheduler, mpy-cross, tests] - if: ${{ needs.scheduler.outputs.boards-esp != '[]' }} - uses: ./.github/workflows/build-boards.yml - secrets: inherit - with: - platform: esp - boards: ${{ needs.scheduler.outputs.boards-esp }} - cp-version: ${{ needs.scheduler.outputs.cp-version }} - - riscv: + ports: needs: [scheduler, mpy-cross, tests] - if: ${{ needs.scheduler.outputs.boards-riscv != '[]' }} + if: needs.scheduler.outputs.ports != '{}' uses: ./.github/workflows/build-boards.yml secrets: inherit + strategy: + fail-fast: false + matrix: + port: ${{ fromJSON(needs.scheduler.outputs.ports).ports }} with: - platform: riscv - boards: ${{ needs.scheduler.outputs.boards-riscv }} + boards: ${{ toJSON(fromJSON(needs.scheduler.outputs.ports)[matrix.port]) }} cp-version: ${{ needs.scheduler.outputs.cp-version }} diff --git a/.github/workflows/create_website_pr.yml b/.github/workflows/create-website-pr.yml similarity index 100% rename from .github/workflows/create_website_pr.yml rename to .github/workflows/create-website-pr.yml diff --git a/.github/workflows/build-boards-custom.yml b/.github/workflows/custom-board-build.yml similarity index 89% rename from .github/workflows/build-boards-custom.yml rename to .github/workflows/custom-board-build.yml index df8197d9ae6bf..aa8044b0d2f52 100644 --- a/.github/workflows/build-boards-custom.yml +++ b/.github/workflows/custom-board-build.yml @@ -32,16 +32,6 @@ run-name: ${{ inputs.board }}-${{ inputs.language }}-${{ inputs.version }}${{ in jobs: build: runs-on: ubuntu-22.04 - env: - PLATFORM_atmel-samd: arm - PLATFORM_broadcom: aarch - PLATFORM_cxd56: arm - PLATFORM_espressif: esp - PLATFORM_litex: riscv - PLATFORM_mimxrt10xx: arm - PLATFORM_nrf: arm - PLATFORM_raspberrypi: arm - PLATFORM_stm: arm steps: - name: Set up repository run: | @@ -70,7 +60,7 @@ jobs: uses: ./.github/actions/deps/external with: action: cache - platform: ${{ env[format('PLATFORM_{0}', steps.set-up-port.outputs.port)] }} + port: ${{ steps.set-up-port.outputs.port }} - name: Set up mpy-cross if: steps.set-up-submodules.outputs.frozen == 'True' uses: ./.github/actions/mpy_cross diff --git a/.github/workflows/notify.yml b/.github/workflows/notify-on-issue-label.yml similarity index 100% rename from .github/workflows/notify.yml rename to .github/workflows/notify-on-issue-label.yml diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 12e8d2ef74e18..44146662f2f9f 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -9,7 +9,7 @@ on: jobs: run: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: @@ -31,7 +31,7 @@ jobs: - name: Set up python uses: actions/setup-python@v4 with: - python-version: 3.x + python-version: 3.8 - name: Set up submodules uses: ./.github/actions/deps/submodules with: diff --git a/ports/atmel-samd/Makefile b/ports/atmel-samd/Makefile index 68beeaf129942..0ac326d3ccf49 100644 --- a/ports/atmel-samd/Makefile +++ b/ports/atmel-samd/Makefile @@ -367,7 +367,7 @@ $(BUILD)/firmware.elf: $(OBJ) $(GENERATED_LD_FILE) $(STEPECHO) "LINK $@" $(Q)echo $(OBJ) > $(BUILD)/firmware.objs $(Q)$(CC) -o $@ $(LDFLAGS) @$(BUILD)/firmware.objs -Wl,--start-group $(LIBS) -Wl,--end-group - $(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(GENERATED_LD_FILE) + $(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(GENERATED_LD_FILE) $(BUILD) $(BUILD)/firmware.bin: $(BUILD)/firmware.elf $(STEPECHO) "Create $@" diff --git a/ports/espressif/Makefile b/ports/espressif/Makefile index 1c79691b337e0..ea70f320cb18f 100644 --- a/ports/espressif/Makefile +++ b/ports/espressif/Makefile @@ -434,7 +434,7 @@ $(BUILD)/firmware.elf: $(OBJ) | esp-idf-stamp $(BUILD)/circuitpython-firmware.bin: $(BUILD)/firmware.elf | tools/build_memory_info.py $(STEPECHO) "Create $@" $(Q)esptool.py --chip $(IDF_TARGET) elf2image $(FLASH_FLAGS) --elf-sha256-offset 0xb0 -o $@ $^ - $(Q)$(PYTHON) tools/build_memory_info.py $< $(BUILD)/esp-idf/sdkconfig $@ + $(Q)$(PYTHON) tools/build_memory_info.py $< $(BUILD)/esp-idf/sdkconfig $@ $(BUILD) $(BUILD)/firmware.bin: $(BUILD)/circuitpython-firmware.bin | esp-idf-stamp $(Q)$(PYTHON) ../../tools/join_bins.py $@ $(BOOTLOADER_OFFSET) $(BUILD)/esp-idf/bootloader/bootloader.bin $(PARTITION_TABLE_OFFSET) $(BUILD)/esp-idf/partition_table/partition-table.bin $(FIRMWARE_OFFSET) $(BUILD)/circuitpython-firmware.bin diff --git a/ports/espressif/tools/build_memory_info.py b/ports/espressif/tools/build_memory_info.py index df9d6a801e15c..a11bef82bcc7a 100644 --- a/ports/espressif/tools/build_memory_info.py +++ b/ports/espressif/tools/build_memory_info.py @@ -6,13 +6,12 @@ # SPDX-License-Identifier: MIT import csv +import json import os -import re import sys from elftools.elf.elffile import ELFFile -print() internal_memory = { "esp32": [ @@ -117,8 +116,12 @@ def find_region(start_address): # This file is the bin used_flash = os.stat(sys.argv[3]).st_size - free_flash = firmware_region - used_flash + +with open(f"{sys.argv[4]}/firmware.size.json", "w") as f: + json.dump({"used_flash": used_flash, "firmware_region": firmware_region}, f) + +print() print( "{:7} bytes used, {:7} bytes free in flash firmware space out of {} bytes ({}kB).".format( used_flash, free_flash, firmware_region, firmware_region / 1024 diff --git a/ports/litex/Makefile b/ports/litex/Makefile index efe99dc5b42e5..5556dea11da17 100644 --- a/ports/litex/Makefile +++ b/ports/litex/Makefile @@ -136,7 +136,7 @@ all: $(BUILD)/firmware.bin $(BUILD)/firmware.dfu $(BUILD)/firmware.elf: $(OBJ) $(STEPECHO) "LINK $@" $(Q)$(CC) -o $@ $(LDFLAGS) $^ -Wl,--start-group $(LIBS) -Wl,--end-group - $(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(LD_FILE) + $(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(LD_FILE) $(BUILD) $(BUILD)/firmware.bin: $(BUILD)/firmware.elf $(STEPECHO) "Create $@" diff --git a/ports/nrf/Makefile b/ports/nrf/Makefile index 972b549f61eea..2267a582a498a 100755 --- a/ports/nrf/Makefile +++ b/ports/nrf/Makefile @@ -212,7 +212,7 @@ $(BUILD)/firmware.elf: $(OBJ) $(GENERATED_LD_FILE) $(STEPECHO) "LINK $@" $(Q)echo $(OBJ) > $(BUILD)/firmware.objs $(Q)$(CC) -o $@ $(LDFLAGS) @$(BUILD)/firmware.objs -Wl,--start-group $(LIBS) -Wl,--end-group - $(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(GENERATED_LD_FILE) + $(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(GENERATED_LD_FILE) $(BUILD) $(BUILD)/firmware.bin: $(BUILD)/firmware.elf $(STEPECHO) "Create $@" diff --git a/ports/raspberrypi/Makefile b/ports/raspberrypi/Makefile index ee0356b85244c..d4400317845bf 100644 --- a/ports/raspberrypi/Makefile +++ b/ports/raspberrypi/Makefile @@ -417,7 +417,7 @@ $(BUILD)/firmware.elf: $(OBJ) $(LINK_LD) $(Q)echo $(OBJ) > $(BUILD)/firmware.objs $(Q)echo $(PICO_LDFLAGS) > $(BUILD)/firmware.ldflags $(Q)$(CC) -o $@ $(CFLAGS) @$(BUILD)/firmware.ldflags -Wl,-T,$(LINK_LD) -Wl,-Map=$@.map -Wl,-cref -Wl,--gc-sections @$(BUILD)/firmware.objs -Wl,-lc - $(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(LINK_LD) + $(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(LINK_LD) $(BUILD) $(BUILD)/firmware.bin: $(BUILD)/firmware.elf $(STEPECHO) "Create $@" diff --git a/ports/stm/Makefile b/ports/stm/Makefile index 094bee7aef1fa..325b13b1bc2da 100755 --- a/ports/stm/Makefile +++ b/ports/stm/Makefile @@ -268,7 +268,7 @@ $(BUILD)/firmware.elf: $(OBJ) $(STEPECHO) "LINK $@" $(Q)echo $^ > $(BUILD)/firmware.objs $(Q)$(CC) -o $@ $(LDFLAGS) @$(BUILD)/firmware.objs -Wl,--start-group $(LIBS) -Wl,--end-group - $(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(LD_FILE) + $(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(LD_FILE) $(BUILD) $(BUILD)/firmware.bin: $(BUILD)/firmware.elf $(STEPECHO) "Create $@" diff --git a/tools/build_memory_info.py b/tools/build_memory_info.py index 722c548a1c281..09d2e72a6416f 100755 --- a/tools/build_memory_info.py +++ b/tools/build_memory_info.py @@ -7,6 +7,8 @@ import re import sys +import json + # Handle size constants with K or M suffixes (allowed in .ld but not in Python). K_PATTERN = re.compile(r"([0-9]+)[kK]") @@ -15,11 +17,10 @@ M_PATTERN = re.compile(r"([0-9]+)[mM]") M_REPLACE = r"(\1*1024*1024)" -print() - text = 0 data = 0 bss = 0 + # stdin is the linker output. for line in sys.stdin: # Uncomment to see linker output. @@ -29,6 +30,7 @@ text, data, bss = map(int, line.split()[:3]) regions = {} + # This file is the linker script. with open(sys.argv[1], "r") as f: for line in f: @@ -51,6 +53,11 @@ free_flash = firmware_region - used_flash used_ram = data + bss free_ram = ram_region - used_ram + +with open(f"{sys.argv[2]}/firmware.size.json", "w") as f: + json.dump({"used_flash": used_flash, "firmware_region": firmware_region}, f) + +print() print( "{} bytes used, {} bytes free in flash firmware space out of {} bytes ({}kB).".format( used_flash, free_flash, firmware_region, firmware_region / 1024 diff --git a/tools/build_release_files.py b/tools/build_release_files.py index a4f28f1c1cc34..341ae2d67f0fc 100755 --- a/tools/build_release_files.py +++ b/tools/build_release_files.py @@ -11,6 +11,7 @@ import shutil import build_board_info as build_info import time +import json sys.path.append("../docs") from shared_bindings_matrix import get_settings_from_makefile @@ -18,7 +19,7 @@ for port in build_info.SUPPORTED_PORTS: result = subprocess.run("rm -rf ../ports/{port}/build*".format(port=port), shell=True) -PARALLEL = "-j 5" +PARALLEL = "-j 4" if "GITHUB_ACTION" in os.environ: PARALLEL = "-j 2" @@ -29,6 +30,11 @@ sha, version = build_info.get_version_info() +build_all = os.environ.get("GITHUB_EVENT_NAME") != "pull_request" + +LANGUAGE_FIRST = "en_US" +LANGUAGE_THRESHOLD = 10 * 1024 + languages = build_info.get_languages() all_languages = build_info.get_languages(list_all=True) @@ -44,6 +50,9 @@ board_info = all_boards[board] board_settings = get_settings_from_makefile("../ports/" + board_info["port"], board) + languages.remove(LANGUAGE_FIRST) + languages.insert(0, LANGUAGE_FIRST) + for language in languages: bin_directory = "../bin/{board}/{language}".format(board=board, language=language) os.makedirs(bin_directory, exist_ok=True) @@ -130,4 +139,16 @@ # Flush so we will see something before 10 minutes has passed. print(flush=True) + if (not build_all) and (language is LANGUAGE_FIRST) and (exit_status is 0): + try: + with open( + f"../ports/{board_info['port']}/{build_dir}/firmware.size.json", "r" + ) as f: + firmware = json.load(f) + if firmware["used_flash"] + LANGUAGE_THRESHOLD < firmware["firmware_region"]: + print("Skipping languages") + break + except FileNotFoundError: + pass + sys.exit(exit_status) diff --git a/tools/ci_changes_per_commit.py b/tools/ci_changes_per_commit.py index d5f621d263451..e83a702b22a98 100644 --- a/tools/ci_changes_per_commit.py +++ b/tools/ci_changes_per_commit.py @@ -167,8 +167,8 @@ def get_commit_depth_and_check_suite(query_commits): def get_bad_check_runs(query_check_runs): + bad_runs = {} more_pages = True - bad_runs_by_matrix = {} run_types = ["failed", "incomplete"] @@ -184,13 +184,12 @@ def get_bad_check_runs(query_check_runs): for check_run in check_runs[run_type]["nodes"]: name = check_run["name"] - res_matrix = regex_matrix.search(name) - if res_matrix: - matrix = name.split(" /", 1)[0] - matrix_job = name.split(" (", 1)[1][:-1] - bad_runs_by_matrix.setdefault(matrix, []).append(matrix_job) + if name.startswith("ports") or regex_matrix.search(name): + matrix = name.split(" ", 1)[0] + matrix_job = name.rsplit(" (", 1)[1][:-1] + bad_runs.setdefault(matrix, []).append(matrix_job) elif name != "scheduler": - bad_runs_by_matrix[name] = True + bad_runs[name] = True else: return {} @@ -200,7 +199,7 @@ def get_bad_check_runs(query_check_runs): query_check_runs.variables["include" + run_type_camel] = True more_pages = True - return bad_runs_by_matrix + return bad_runs def set_commit(commit): diff --git a/tools/ci_fetch_deps.py b/tools/ci_fetch_deps.py index 3b495f8758ce2..1b292b9d734bf 100644 --- a/tools/ci_fetch_deps.py +++ b/tools/ci_fetch_deps.py @@ -78,13 +78,13 @@ def main(): submodules = ["extmod/ulab", "lib/", "tools/"] elif TARGET == "tests": submodules = ["extmod/ulab", "lib/", "tools/"] - elif TARGET == "build-doc": + elif TARGET == "docs": # used in .readthedocs.yml to generate RTD submodules = ["extmod/ulab"] submodules_tags = ["frozen/"] elif TARGET == "mpy-cross" or TARGET == "mpy-cross-mac": submodules = ["tools/"] # for huffman - elif TARGET == "build-windows": + elif TARGET == "windows": # This builds one board from a number of ports so fill out a bunch of submodules submodules = ["extmod/ulab", "lib/", "tools/", "ports/", "data/nvm.toml"] elif TARGET == "website": diff --git a/tools/ci_set_matrix.py b/tools/ci_set_matrix.py index 02a1cbee7313c..fe823ac0303ba 100755 --- a/tools/ci_set_matrix.py +++ b/tools/ci_set_matrix.py @@ -42,18 +42,6 @@ all_ports_all_boards, ) -PORT_TO_JOB = { - "atmel-samd": "atmel", - "broadcom": "aarch", - "cxd56": "arm", - "espressif": "esp", - "litex": "riscv", - "mimxrt10xx": "arm", - "nrf": "arm", - "raspberrypi": "arm", - "stm": "arm", -} - IGNORE = [ "tools/ci_set_matrix.py", "tools/ci_check_duplicate_usb_vid_pid.py", @@ -109,7 +97,7 @@ def set_output(name: str, value): print(f"Would set GitHub actions output {name} to '{value}'") -def set_boards_to_build(build_all: bool): +def set_boards(build_all: bool): if last_failed_jobs.get("mpy-cross") or last_failed_jobs.get("tests"): build_all = True @@ -224,18 +212,13 @@ def get_settings(board): boards_to_build = all_board_ids break - # Split boards by job - job_to_boards = {"aarch": [], "arm": [], "atmel": [], "esp": [], "riscv": []} - # Append previously failed boards - for job in job_to_boards: - if job in last_failed_jobs: - for board in last_failed_jobs[job]: - boards_to_build.add(board) + boards_to_build.update(last_failed_jobs.get("ports") or []) - build_boards = bool(boards_to_build) - print("Building boards:", build_boards) - set_output("build-boards", build_boards) + print("Building boards:", bool(boards_to_build)) + + # Split boards by port + port_to_boards_to_build = {} # Append boards according to job for board in sorted(boards_to_build): @@ -244,17 +227,19 @@ def get_settings(board): # if this happens it's not in `board_to_port`. if not port: continue - job_to_boards[PORT_TO_JOB[port]].append(board) + port_to_boards_to_build.setdefault(port, []).append(board) print(" ", board) - # Set the step outputs for each job - for job in job_to_boards: - set_output(f"boards-{job}", json.dumps(job_to_boards[job])) + if port_to_boards_to_build: + port_to_boards_to_build["ports"] = sorted(list(port_to_boards_to_build.keys())) + # Set the step outputs + set_output("ports", json.dumps(port_to_boards_to_build)) -def set_docs_to_build(build_doc: bool): + +def set_docs(build_doc: bool): if not build_doc: - if last_failed_jobs.get("build-doc"): + if last_failed_jobs.get("docs"): build_doc = True else: doc_pattern = re.compile(PATTERN_DOCS) @@ -277,12 +262,12 @@ def set_docs_to_build(build_doc: bool): # Set the step outputs print("Building docs:", build_doc) - set_output("build-doc", build_doc) + set_output("docs", build_doc) -def set_windows_to_build(build_windows): +def set_windows(build_windows: bool): if not build_windows: - if last_failed_jobs.get("build-windows"): + if last_failed_jobs.get("windows"): build_windows = True else: for file in changed_files: @@ -296,23 +281,22 @@ def set_windows_to_build(build_windows): # Set the step outputs print("Building windows:", build_windows) - set_output("build-windows", build_windows) - - -def check_changed_files(): - if not changed_files: - print("Building all docs/boards") - return True - else: - print("Adding docs/boards to build based on changed files") - return False + set_output("windows", build_windows) def main(): - build_all = check_changed_files() - set_docs_to_build(build_all) - set_windows_to_build(build_all) - set_boards_to_build(build_all) + # Build all if no changed files + build_all = not changed_files + print( + "Building all docs/boards" + if build_all + else "Adding docs/boards to build based on changed files" + ) + + # Set jobs + set_docs(build_all) + set_windows(build_all) + set_boards(build_all) if __name__ == "__main__":