diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 9f0f0bb6..36025d5a 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -8,8 +8,6 @@ on:
 
 env:
   BUILDDIR: gh
-  DL_DIR: ${{ vars.BUILD_DL_DIR }}
-  SSTATE_DIR: ${{ vars.BUILD_SSTATE_DIR }}
 
 jobs:
   prepare:
@@ -17,6 +15,11 @@ jobs:
     steps:
       - name: Clean up
         run: rm -fR $BUILDDIR
+      - uses: actions/checkout@v3
+      - name: cache metadata layers
+        run: ./ci/update-repos
+        env:
+          LAYERCACHE: ${{ vars.PERSISTENT_DIR }}/layers
 
   build:
     needs: [prepare]
@@ -96,10 +99,14 @@ jobs:
       # the container image needs to be effectively hardcoded, it seems.
       image: ghcr.io/theyoctojester/devcontainer-base-yoep:main
       volumes:
-        - ${{ vars.KAS_DL_DIR }}:${{ vars.BUILD_DL_DIR }}
-        - ${{ vars.KAS_SSTATE_DIR }}:${{ vars.BUILD_SSTATE_DIR }}
+        - ${{ vars.PERSISTENT_DIR }}/downloads:${{ vars.BUILD_DL_DIR }}
+        - ${{ vars.PERSISTENT_DIR }}/sstate-cache:${{ vars.BUILD_SSTATE_DIR }}
+        - ${{ vars.PERSISTENT_DIR }}/layers:${{ vars.BUILD_LAYERCACHE_DIR }}
       options: --user ${{ vars.KAS_UID }}:${{ vars.KAS_GID }}
     steps:
-      - uses: actions/checkout@v3
       - name: enter build dir and build
-        run: mkdir -p $BUILDDIR/${{ matrix.board }} && cd $BUILDDIR/${{ matrix.board }} && kas build ../../kas/${{ matrix.board }}.yml
\ No newline at end of file
+        env:
+          DL_DIR: ${{ vars.BUILD_DL_DIR}}
+          SSTATE_DIR: ${{ vars.BUILD_SSTATE_DIR}}
+          KAS_REPO_REF_DIR: ${{ vars.BUILD_LAYERCACHE_DIR}}
+        run: mkdir -p $BUILDDIR/${{ matrix.board }} && cd $BUILDDIR/${{ matrix.board }} && kas build ../../kas/${{ matrix.board }}.yml
diff --git a/ci/update-repos b/ci/update-repos
new file mode 100755
index 00000000..840eae28
--- /dev/null
+++ b/ci/update-repos
@@ -0,0 +1,65 @@
+#! /usr/bin/env python3
+
+# Update clones of the repositories we need in KAS_REPO_REF_DIR to speed up fetches
+
+import sys
+import os
+import shutil
+import subprocess
+import pathlib
+
+def repo_shortname(url):
+    # Taken from Kas (Repo.__getattr__) to ensure the logic is right
+    from urllib.parse import urlparse
+    url = urlparse(url)
+    return ('{url.netloc}{url.path}'
+            .format(url=url)
+            .replace('@', '.')
+            .replace(':', '.')
+            .replace('/', '.')
+            .replace('*', '.'))
+
+repositories = (
+    "https://git.yoctoproject.org/git/poky",
+    "https://git.openembedded.org/meta-openembedded",
+    "https://git.yoctoproject.org/git/meta-virtualization",
+    "https://github.com/kraj/meta-clang",
+    "https://github.com/mendersoftware/meta-mender",
+    "https://github.com/agherzan/meta-raspberrypi",
+    "https://github.com/OE4T/meta-tegra.git",
+    "https://github.com/Freescale/meta-freescale",
+    "https://github.com/Freescale/meta-freescale-distro",
+    "https://github.com/Freescale/meta-freescale-3rdparty",
+    "https://git.openembedded.org/openembedded-core",
+    "https://git.openembedded.org/bitbake",
+    "https://git.yoctoproject.org/git/meta-yocto",
+)
+
+if __name__ == "__main__":
+    if "LAYERCACHE" not in os.environ:
+        print("LAYERCACHE needs to be set")
+        sys.exit(1)
+
+    base_repodir = pathlib.Path(os.environ["LAYERCACHE"])
+    failed = False
+
+    for repo in repositories:
+        repodir = base_repodir / repo_shortname(repo)
+
+        if "CI_CLEAN_REPOS" in os.environ:
+            print("Cleaning %s..." % repo)
+            shutil.rmtree(repodir, ignore_errors=True)
+
+        if repodir.exists():
+            try:
+                print("Updating %s..." % repo)
+                subprocess.run(["git", "-C", repodir, "-c", "gc.autoDetach=false", "fetch"], check=True)
+            except subprocess.CalledProcessError as e:
+                print(e)
+                failed = True
+        else:
+            print("Cloning %s..." % repo)
+            subprocess.run(["git", "clone", "--bare", repo, repodir], check=True)
+
+    if failed:
+        sys.exit(128)