Remote sstate-cache mirror & first kanto migration steps

.config.yaml

@@ -52,6 +52,9 @@ repos:
   meta-rauc:
     url: "https://github.com/rauc/meta-rauc.git"
     refspec: kirkstone
+  meta-kanto:
+    url: "https://github.com/eclipse-kanto/meta-kanto.git"
+    refspec: kirkstone
   meta-virtualization:
     url: "https://git.yoctoproject.org/meta-virtualization"
     refspec: kirkstone

.devcontainer/post-attach.sh

@@ -27,15 +27,25 @@ sudo chmod 0660 /dev/kvm
 # AZURE_STORAGE_ACCESS_KEY="<Access key>"
 # AZURE_STORAGE_ACCOUNT="sdvyocto" // Replace with your storage name
 # AZURE_STORAGE_ACCOUNT_CONTAINER="yocto-sstate-cache" // Replace with your storage container name
-if [ -n "${AZURE_MOUNT_POINT}" ] && [ -n "${AZURE_STORAGE_ACCOUNT_CONTAINER}" ]; then
-    [ -d ${AZURE_MOUNT_POINT} ] || mkdir -p ${AZURE_MOUNT_POINT}
-    if $(blobfuse2 mount list | grep -q "${AZURE_MOUNT_POINT}"); then
-        echo "Unmounting remote sstate-cache: ${AZURE_MOUNT_POINT}"
-        blobfuse2 --disable-version-check unmount ${AZURE_MOUNT_POINT}
+function azure-mount() {
+    local AZURE_MOUNT_POINT="$1"
+    local AZURE_STORAGE_ACCOUNT_CONTAINER="$2"
+    if [ -n "${AZURE_MOUNT_POINT}" ] && [ -n "${AZURE_STORAGE_ACCOUNT_CONTAINER}" ]; then
+        [ -d ${AZURE_MOUNT_POINT} ] || mkdir -p ${AZURE_MOUNT_POINT}
+        if $(blobfuse2 mount list | grep -q "${AZURE_MOUNT_POINT}"); then
+            echo "Already mounted: ${AZURE_MOUNT_POINT}"
+        else
+            TEMPD=$(mktemp -d)
+            echo "Mounting remote Azure Storage ${AZURE_STORAGE_ACCOUNT_CONTAINER} to local path: ${AZURE_MOUNT_POINT}"
+            blobfuse2 --disable-version-check mount ${AZURE_MOUNT_POINT} --log-level LOG_DEBUG --use-https=true --tmp-path=$TEMPD --container-name=${AZURE_STORAGE_ACCOUNT_CONTAINER} || true
+        fi
+    else
+        echo "Warning: No AZURE_MOUNT_POINT or no AZURE_STORAGE_ACCOUNT_CONTAINER set for mounting."
     fi
-    TEMPD=$(mktemp -d)
-    echo "Mounting remote sstate-cache on Azure Storage Account ${AZURE_STORAGE_ACCOUNT_CONTAINER} to local path: ${AZURE_MOUNT_POINT}"
-    blobfuse2 --disable-version-check mount ${AZURE_MOUNT_POINT} --log-level LOG_DEBUG --use-https=true --tmp-path=$TEMPD --container-name=${AZURE_STORAGE_ACCOUNT_CONTAINER} || true
-else
-    echo "Warning: No AZURE_MOUNT_POINT or no AZURE_STORAGE_ACCOUNT_CONTAINER set for remote sstate-cache."
-fi
+}
+
+# First arg: local mount point
+# Second arg: Name of Azure container
+azure-mount "azure-sstate-cache" "yocto-sstate-cache"
+azure-mount "azure-downloads-cache" "downloads"
+

.github/workflows/dryrun.yml

@@ -24,7 +24,7 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v3
     - name: Install tools
-      run: sudo apt-get install -y --no-install-recommends socat file gawk wget git diffstat unzip texinfo gcc build-essential chrpath socat cpio python3 python3-pip python3-pexpect xz-utils debianutils iputils-ping python3-git python3-jinja2 libegl1-mesa libsdl1.2-dev pylint3 xterm python3-subunit mesa-common-dev zstd liblz4-tool tmux mc skopeo fdisk ruby-full jq libvirt-clients libvirt-daemon-system qemu-system-x86 qemu-system-arm qemu-kvm squashfs-tools rauc python3-newt ca-certificates curl gnupg lsb-release
+      run: sudo apt-get install -y --no-install-recommends socat file gawk wget git diffstat unzip texinfo gcc build-essential chrpath socat cpio python3 python3-pip python3-pexpect xz-utils debianutils iputils-ping python3-git python3-jinja2 libegl1-mesa libsdl1.2-dev pylint xterm python3-subunit mesa-common-dev zstd liblz4-tool tmux mc skopeo fdisk ruby-full jq libvirt-clients libvirt-daemon-system qemu-system-x86 qemu-system-arm qemu-kvm squashfs-tools rauc python3-newt ca-certificates curl gnupg lsb-release
     - name: Install kas
       run: sudo pip3 install kas
     - name: DryRun sdv-image-all

.github/workflows/leda-utils.yml

@@ -24,7 +24,7 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v3
     - name: Install tools
-      run: sudo apt-get install -y --no-install-recommends socat file gawk wget git diffstat unzip texinfo gcc build-essential chrpath socat cpio python3 python3-pip python3-pexpect xz-utils debianutils iputils-ping python3-git python3-jinja2 libegl1-mesa libsdl1.2-dev pylint3 xterm python3-subunit mesa-common-dev zstd liblz4-tool tmux mc skopeo fdisk ruby-full jq libvirt-clients libvirt-daemon-system qemu-system-x86 qemu-system-arm qemu-kvm squashfs-tools rauc python3-newt ca-certificates curl gnupg lsb-release
+      run: sudo apt-get install -y --no-install-recommends socat file gawk wget git diffstat unzip texinfo gcc build-essential chrpath socat cpio python3 python3-pip python3-pexpect xz-utils debianutils iputils-ping python3-git python3-jinja2 libegl1-mesa libsdl1.2-dev pylint xterm python3-subunit mesa-common-dev zstd liblz4-tool tmux mc skopeo fdisk ruby-full jq libvirt-clients libvirt-daemon-system qemu-system-x86 qemu-system-arm qemu-kvm squashfs-tools rauc python3-newt ca-certificates curl gnupg lsb-release
     - name: Install kas
       run: sudo pip3 install kas
     - name: Start BitBake Hash Equivalence Server

.gitignore

@@ -3,6 +3,7 @@ build/
 
 # Metalayer folders are created by running "kas build" and we don't
 # want to commit them as git submodules.
+meta-kanto
 meta-openembedded
 meta-rauc
 meta-rauc-community
@@ -11,3 +12,6 @@ poky
 
 # BitBake Hash Equivalence Server database
 hashserv.db*
+azure-sstate-cache
+azure-downloads-cache
+bitbake

README.md

@@ -46,6 +46,9 @@ This layer depends on
 - meta-rauc:
   - URI: https://github.com/rauc/meta-rauc.git
   - Branch: kirkstone
+- meta-kanto:
+  - URI: https://github.com/eclipse-kanto/meta-kanto.git
+  - Branch: kirkstone
 - meta-virtualization:
   - URI: https://git.yoctoproject.org/meta-virtualization
   - Branch: kirkstone

docs/manual-bitbake-dryrun.md

@@ -50,4 +50,4 @@ Dry-run a build of one of the Leda recipes:
 
 Perform a real build of one of the targets, e.g. the Eclipse Kanto Container Management recipe:
 
-    DISTRO=leda bitbake kanto-container-management
+    DISTRO=leda bitbake container-management

kas/.config-honister.yaml

@@ -49,6 +49,9 @@ repos:
   meta-rauc:
     url: "https://github.com/rauc/meta-rauc.git"
     refspec: honister
+  meta-kanto:
+    url: "https://github.com/eclipse-kanto/meta-kanto.git"
+    refspec: main
   meta-virtualization:
     url: "https://git.yoctoproject.org/meta-virtualization"
     refspec: honister

kas/.config-kirkstone.yaml

@@ -29,6 +29,9 @@ local_conf_header:
     INHERIT:remove = " archiver"
     INHERIT:remove = " cve-check"
     INHIBIT_PACKAGE_DEBUG_SPLIT = "1"
+  sign_conf: |
+    RAUC_KEY_FILE="${TOPDIR}/../examples/rauc/development-1.key.pem"
+    RAUC_CERT_FILE="${TOPDIR}/../examples/rauc/development-1.cert.pem"
 repos:
   # This entry includes the repository where the config file is located
   # to the bblayers.conf:
@@ -49,6 +52,9 @@ repos:
   meta-rauc:
     url: "https://github.com/rauc/meta-rauc.git"
     refspec: kirkstone
+  meta-kanto:
+    url: "https://github.com/eclipse-kanto/meta-kanto.git"
+    refspec: kirkstone
   meta-virtualization:
     url: "https://git.yoctoproject.org/meta-virtualization"
     refspec: kirkstone

kas/.config-master.yaml

@@ -49,6 +49,9 @@ repos:
   meta-rauc:
     url: "https://github.com/rauc/meta-rauc.git"
     refspec: master
+  meta-kanto:
+    url: "https://github.com/eclipse-kanto/meta-kanto.git"
+    refspec: main
   meta-virtualization:
     url: "https://git.yoctoproject.org/meta-virtualization"
     refspec: master

kas/mirrors.yaml

@@ -21,5 +21,22 @@ header:
 # The machine as it is written into the `local.conf` of bitbake.
 local_conf_header:
   mirror: |
-    BB_HASHSERVE = "127.0.0.1:1234"
+    BB_HASHSERVE = "20.238.216.74:1234"
+    MIRROR_SERVER = "https://sdvyocto.blob.core.windows.net/downloads/"
     SSTATE_MIRRORS = "file://.* https://sdvyocto.blob.core.windows.net/yocto-sstate-cache/sstate-cache/PATH"
+    BB_FETCH_PREMIRRORONLY = "0"
+    INHERIT += "own-mirrors"
+    SOURCE_MIRROR_URL = "${MIRROR_SERVER}"
+    UNINATIVE_URL = "${MIRROR_SERVER}"
+    PREMIRRORS ?= "\
+      bzr://.*/.\* ${MIRROR_SERVER} \
+      cvs://.*/.\* ${MIRROR_SERVER} \
+      git://.*/.\* ${MIRROR_SERVER} \
+      hg://.*/.\*  ${MIRROR_SERVER} \
+      osc://.*/.\* ${MIRROR_SERVER} \
+      p4://.*/.\*  ${MIRROR_SERVER} \
+      svn://.*/.\* ${MIRROR_SERVER}"
+    MIRRORS =+ "\
+      ftp://.*/.\*   ${MIRROR_SERVER} \
+      http://.*/.\*  ${MIRROR_SERVER} \
+      https://.*/.\* ${MIRROR_SERVER}"