CI: Enhancements & Refactoring

.github/actions/deps/external/action.yml

@@ -9,54 +9,31 @@ inputs:
     - cache
     - restore
 
-  platform:
+  port:
     required: false
     default: none
-    type: choice
-    options:
-    - arm
-    - aarch
-    - esp
-    - riscv
-    - none
+    type: string
 
 runs:
   using: composite
   steps:
-    # aarch
-    - name: Get aarch toolchain
-      if: inputs.platform == 'aarch'
-      run: |
-        wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
-        sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
-        sudo apt-get install -y mtools
-      shell: bash
-    - name: Install mkfs.fat
-      if: inputs.platform == 'aarch'
-      run: |
-        wget https://github.com/dosfstools/dosfstools/releases/download/v4.2/dosfstools-4.2.tar.gz
-        tar -xaf dosfstools-4.2.tar.gz
-        cd dosfstools-4.2
-        ./configure
-        make -j 2
-        cd src
-        echo >> $GITHUB_PATH $(pwd)
-      shell: bash
-
     # arm
     - name: Get arm toolchain
-      if: inputs.platform == 'aarch' || inputs.platform == 'arm'
+      if: >-
+        inputs.port != 'none' &&
+        inputs.port != 'litex' &&
+        inputs.port != 'espressif'
       uses: carlosperate/arm-none-eabi-gcc-action@v1
       with:
         release: '10-2020-q4'
 
-    # esp
-    - name: Get esp toolchain
-      if: inputs.platform == 'esp'
+    # espressif
+    - name: Get espressif toolchain
+      if: inputs.port == 'espressif'
       run: sudo apt-get install -y ninja-build
       shell: bash
     - name: Install IDF tools
-      if: inputs.platform == 'esp'
+      if: inputs.port == 'espressif'
       run: |
         echo "Installing ESP-IDF tools"
         $IDF_PATH/tools/idf_tools.py --non-interactive install required
@@ -66,24 +43,16 @@ runs:
         rm -rf $IDF_TOOLS_PATH/dist
       shell: bash
     - name: Set environment
-      if: inputs.platform == 'esp'
+      if: inputs.port == 'espressif'
       run: |
         source $IDF_PATH/export.sh
         echo >> $GITHUB_ENV "IDF_PYTHON_ENV_PATH=$IDF_PYTHON_ENV_PATH"
         echo >> $GITHUB_PATH "$PATH"
       shell: bash
 
-    # riscv
-    - name: Get riscv toolchain
-      if: inputs.platform == 'riscv'
-      run: |
-        wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
-        sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
-      shell: bash
-
     # common
     - name: Cache python dependencies
-      if: inputs.platform != 'esp'
+      if: inputs.port != 'espressif'
       uses: ./.github/actions/deps/python
       with:
         action: ${{ inputs.action }}

.github/actions/deps/ports/action.yml

@@ -19,10 +19,18 @@ runs:
         if [ -z $PORT ]; then (exit 1); else echo >> $GITHUB_OUTPUT "port=$PORT"; fi
       shell: bash
 
-    - name: Set up espressif port
+    - name: Set up broadcom
+      if: steps.board-to-port.outputs.port == 'broadcom'
+      uses: ./.github/actions/deps/ports/broadcom
+
+    - name: Set up espressif
       if: steps.board-to-port.outputs.port == 'espressif'
       uses: ./.github/actions/deps/ports/espressif
 
-    - name: Set up nrf port
+    - name: Set up litex
+      if: steps.board-to-port.outputs.port == 'litex'
+      uses: ./.github/actions/deps/ports/litex
+
+    - name: Set up nrf
       if: steps.board-to-port.outputs.port == 'nrf'
       uses: ./.github/actions/deps/ports/nrf

.github/actions/deps/ports/broadcom/action.yml

@@ -0,0 +1,21 @@
+name: Fetch broadcom port deps
+
+runs:
+  using: composite
+  steps:
+    - name: Get broadcom toolchain
+      run: |
+        wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
+        sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
+        sudo apt-get install -y mtools
+      shell: bash
+    - name: Install mkfs.fat
+      run: |
+        wget https://github.com/dosfstools/dosfstools/releases/download/v4.2/dosfstools-4.2.tar.gz
+        tar -xaf dosfstools-4.2.tar.gz
+        cd dosfstools-4.2
+        ./configure
+        make -j 2
+        cd src
+        echo >> $GITHUB_PATH $(pwd)
+      shell: bash

.github/actions/deps/ports/litex/action.yml

@@ -0,0 +1,10 @@
+name: Fetch litex port deps
+
+runs:
+  using: composite
+  steps:
+    - name: Get litex toolchain
+      run: |
+        wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
+        sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
+      shell: bash

.github/workflows/build-boards.yml

@@ -3,9 +3,6 @@ name: Build boards
 on:
   workflow_call:
     inputs:
-      platform:
-        required: true
-        type: string
       boards:
         required: true
         type: string
@@ -19,7 +16,7 @@ on:
         required: false
 
 jobs:
-  build:
+  board:
     runs-on: ubuntu-22.04
     env:
       CP_VERSION: ${{ inputs.cp-version }}
@@ -38,6 +35,7 @@ jobs:
       with:
         python-version: 3.x
     - name: Set up port
+      id: set-up-port
       uses: ./.github/actions/deps/ports
       with:
         board: ${{ matrix.board }}
@@ -47,7 +45,7 @@ jobs:
     - name: Set up external
       uses: ./.github/actions/deps/external
       with:
-        platform: ${{ inputs.platform }}
+        port: ${{ steps.set-up-port.outputs.port }}
     - name: Set up mpy-cross
       if: steps.set-up-submodules.outputs.frozen == 'True'
       uses: ./.github/actions/mpy_cross

.github/workflows/build.yml

@@ -18,16 +18,11 @@ concurrency:
 
 jobs:
   scheduler:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     outputs:
-      build-doc: ${{ steps.set-matrix.outputs.build-doc }}
-      build-boards: ${{ steps.set-matrix.outputs.build-boards }}
-      build-windows: ${{ steps.set-matrix.outputs.build-windows }}
-      boards-aarch: ${{ steps.set-matrix.outputs.boards-aarch }}
-      boards-arm: ${{ steps.set-matrix.outputs.boards-arm }}
-      boards-atmel: ${{ steps.set-matrix.outputs.boards-atmel }}
-      boards-esp: ${{ steps.set-matrix.outputs.boards-esp }}
-      boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }}
+      docs: ${{ steps.set-matrix.outputs.docs }}
+      ports: ${{ steps.set-matrix.outputs.ports }}
+      windows: ${{ steps.set-matrix.outputs.windows }}
       cp-version: ${{ steps.set-up-submodules.outputs.version }}
     steps:
     - name: Dump GitHub context
@@ -99,7 +94,7 @@ jobs:
 
   mpy-cross:
     needs: scheduler
-    if: needs.scheduler.outputs.build-boards == 'True'
+    if: needs.scheduler.outputs.ports != '{}'
     uses: ./.github/workflows/build-mpy-cross.yml
     secrets: inherit
     with:
@@ -108,7 +103,7 @@ jobs:
   mpy-cross-mac:
     runs-on: macos-11
     needs: scheduler
-    if: needs.scheduler.outputs.build-boards == 'True'
+    if: needs.scheduler.outputs.ports != '{}'
     env:
       CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
     steps:
@@ -160,10 +155,10 @@ jobs:
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
-  build-doc:
+  docs:
     runs-on: ubuntu-22.04
     needs: scheduler
-    if: needs.scheduler.outputs.build-doc == 'True'
+    if: needs.scheduler.outputs.docs == 'True'
     env:
       CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
     steps:
@@ -218,10 +213,10 @@ jobs:
         [ -z "$TWINE_USERNAME" ] || echo "Uploading dev release to PyPi"
         [ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/*
 
-  build-windows:
+  windows:
     runs-on: windows-2022
     needs: scheduler
-    if: needs.scheduler.outputs.build-windows == 'True'
+    if: needs.scheduler.outputs.windows == 'True'
     env:
       CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
     defaults:
@@ -286,52 +281,15 @@ jobs:
     # ERROR: Platform MINGW64_NT-10.0-17763-x86_64 appears to be unsupported
     # https://github.com/espressif/esp-idf/issues/7062
 
-  aarch:
-    needs: [scheduler, mpy-cross, tests]
-    if: ${{ needs.scheduler.outputs.boards-aarch != '[]' }}
-    uses: ./.github/workflows/build-boards.yml
-    secrets: inherit
-    with:
-      platform: aarch
-      boards: ${{ needs.scheduler.outputs.boards-aarch }}
-      cp-version: ${{ needs.scheduler.outputs.cp-version }}
-
-  arm:
-    needs: [scheduler, mpy-cross, tests]
-    if: ${{ needs.scheduler.outputs.boards-arm != '[]' }}
-    uses: ./.github/workflows/build-boards.yml
-    secrets: inherit
-    with:
-      platform: arm
-      boards: ${{ needs.scheduler.outputs.boards-arm }}
-      cp-version: ${{ needs.scheduler.outputs.cp-version }}
-
-  atmel:
-    needs: [scheduler, mpy-cross, tests]
-    if: ${{ needs.scheduler.outputs.boards-atmel != '[]' }}
-    uses: ./.github/workflows/build-boards.yml
-    secrets: inherit
-    with:
-      platform: arm
-      boards: ${{ needs.scheduler.outputs.boards-atmel }}
-      cp-version: ${{ needs.scheduler.outputs.cp-version }}
-
-  esp:
-    needs: [scheduler, mpy-cross, tests]
-    if: ${{ needs.scheduler.outputs.boards-esp != '[]' }}
-    uses: ./.github/workflows/build-boards.yml
-    secrets: inherit
-    with:
-      platform: esp
-      boards: ${{ needs.scheduler.outputs.boards-esp }}
-      cp-version: ${{ needs.scheduler.outputs.cp-version }}
-
-  riscv:
+  ports:
     needs: [scheduler, mpy-cross, tests]
-    if: ${{ needs.scheduler.outputs.boards-riscv != '[]' }}
+    if: needs.scheduler.outputs.ports != '{}'
     uses: ./.github/workflows/build-boards.yml
     secrets: inherit
+    strategy:
+      fail-fast: false
+      matrix:
+        port: ${{ fromJSON(needs.scheduler.outputs.ports).ports }}
     with:
-      platform: riscv
-      boards: ${{ needs.scheduler.outputs.boards-riscv }}
+      boards: ${{ toJSON(fromJSON(needs.scheduler.outputs.ports)[matrix.port]) }}
       cp-version: ${{ needs.scheduler.outputs.cp-version }}

.github/workflows/build-boards-custom.yml → .github/workflows/custom-board-build.yml

@@ -32,16 +32,6 @@ run-name: ${{ inputs.board }}-${{ inputs.language }}-${{ inputs.version }}${{ in
 jobs:
   build:
     runs-on: ubuntu-22.04
-    env:
-      PLATFORM_atmel-samd: arm
-      PLATFORM_broadcom: aarch
-      PLATFORM_cxd56: arm
-      PLATFORM_espressif: esp
-      PLATFORM_litex: riscv
-      PLATFORM_mimxrt10xx: arm
-      PLATFORM_nrf: arm
-      PLATFORM_raspberrypi: arm
-      PLATFORM_stm: arm
     steps:
     - name: Set up repository
       run: |
@@ -70,7 +60,7 @@ jobs:
       uses: ./.github/actions/deps/external
       with:
         action: cache
-        platform: ${{ env[format('PLATFORM_{0}', steps.set-up-port.outputs.port)] }}
+        port: ${{ steps.set-up-port.outputs.port }}
     - name: Set up mpy-cross
       if: steps.set-up-submodules.outputs.frozen == 'True'
       uses: ./.github/actions/mpy_cross

.github/workflows/run-tests.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   run:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
@@ -31,7 +31,7 @@ jobs:
     - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: 3.x
+        python-version: 3.8
     - name: Set up submodules
       uses: ./.github/actions/deps/submodules
       with:

ports/atmel-samd/Makefile

@@ -367,7 +367,7 @@ $(BUILD)/firmware.elf: $(OBJ) $(GENERATED_LD_FILE)
 	$(STEPECHO) "LINK $@"
 	$(Q)echo $(OBJ) > $(BUILD)/firmware.objs
 	$(Q)$(CC) -o $@ $(LDFLAGS) @$(BUILD)/firmware.objs -Wl,--start-group $(LIBS) -Wl,--end-group
-	$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(GENERATED_LD_FILE)
+	$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(GENERATED_LD_FILE) $(BUILD)
 
 $(BUILD)/firmware.bin: $(BUILD)/firmware.elf
 	$(STEPECHO) "Create $@"

ports/espressif/Makefile

@@ -434,7 +434,7 @@ $(BUILD)/firmware.elf: $(OBJ) | esp-idf-stamp
 $(BUILD)/circuitpython-firmware.bin: $(BUILD)/firmware.elf | tools/build_memory_info.py
 	$(STEPECHO) "Create $@"
 	$(Q)esptool.py --chip $(IDF_TARGET) elf2image $(FLASH_FLAGS) --elf-sha256-offset 0xb0 -o $@ $^
-	$(Q)$(PYTHON) tools/build_memory_info.py $< $(BUILD)/esp-idf/sdkconfig $@
+	$(Q)$(PYTHON) tools/build_memory_info.py $< $(BUILD)/esp-idf/sdkconfig $@ $(BUILD)
 
 $(BUILD)/firmware.bin: $(BUILD)/circuitpython-firmware.bin | esp-idf-stamp
 	$(Q)$(PYTHON) ../../tools/join_bins.py $@ $(BOOTLOADER_OFFSET) $(BUILD)/esp-idf/bootloader/bootloader.bin $(PARTITION_TABLE_OFFSET) $(BUILD)/esp-idf/partition_table/partition-table.bin $(FIRMWARE_OFFSET) $(BUILD)/circuitpython-firmware.bin

ports/espressif/tools/build_memory_info.py

@@ -6,13 +6,12 @@
 # SPDX-License-Identifier: MIT
 
 import csv
+import json
 import os
-import re
 import sys
 
 from elftools.elf.elffile import ELFFile
 
-print()
 
 internal_memory = {
     "esp32": [
@@ -117,8 +116,12 @@ def find_region(start_address):
 
 # This file is the bin
 used_flash = os.stat(sys.argv[3]).st_size
-
 free_flash = firmware_region - used_flash
+
+with open(f"{sys.argv[4]}/firmware.size.json", "w") as f:
+    json.dump({"used_flash": used_flash, "firmware_region": firmware_region}, f)
+
+print()
 print(
     "{:7} bytes used, {:7} bytes free in flash firmware space out of {} bytes ({}kB).".format(
         used_flash, free_flash, firmware_region, firmware_region / 1024