refactor(release): switch release integrity to be dynamic

.bazelrc

@@ -15,7 +15,11 @@ build --embed_label=v1.2.3
 # Mock versioning command to test the --stamp behavior
 build --workspace_status_command="echo BUILD_SCM_VERSION 1.2.3"
 
-common --compilation_mode opt
+# For releasing, use --workspace_status_command and stamp
+# before adding more flags to the release config make sure it does not
+# affect the hashes of /tools. See tools/release.bzl for opt transition
+# add appropriate commandline transition there to match the configuration.
+common:release -c opt
 
 # Load any settings & overrides specific to the current user from `.aspect/bazelrc/user.bazelrc`.
 # This file should appear in `.gitignore` so that settings are not shared with team members. This

.github/workflows/integrity.jq

@@ -0,0 +1,21 @@
+# JQ filter to transform sha256 files to a value we can read from starlark.
+# NB: the sha256 files are expected to be newline-terminated.
+#
+# Input looks like
+# 48552e399a1f2ab97e62ca7fce5783b6214e284330c7555383f43acf82446636 unpack-linux-aarch64\nfd265552bfd236efef519f81ce783322a50d8d7ab5af5d08a713e519cedff87f unpack-linux-x86_64\n
+#
+# Output should look like
+# {
+#  "unpack-linux-aarch64": "48552e399a1f2ab97e62ca7fce5783b6214e284330c7555383f43acf82446636",
+#  "unpack-linux-x86_64": "fd265552bfd236efef519f81ce783322a50d8d7ab5af5d08a713e519cedff87f"
+# }
+
+.
+# Don't end with an empty object
+| rtrimstr("\n")
+| split("\n")
+| map(
+    split(" ")
+    | {"key": .[1], "value": .[0]}
+  )
+| from_entries

.github/workflows/release.yml

@@ -10,47 +10,43 @@ on:
 
 jobs:
   build:
+    # Go cross-compilation works from linux -> any platform
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Mount bazel caches
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/bazel
-            ~/.cache/bazel-repo
-          key: bazel-cache-release-${{ hashFiles('.bazelrc', '.bazelversion', '.bazeliskrc', '**/BUILD', '**/BUILD.bazel', '**/*.bzl', 'WORKSPACE', 'WORKSPACE.bazel', 'WORKSPACE.bzlmod', 'MODULE.bazel') }}
-          restore-keys: bazel-cache-release-
-      - name: bazel test //... (release)
+      - uses: actions/checkout@v4
+      - name: Build Go Binaries
         env:
-          # Bazelisk will download bazel to here
-          XDG_CACHE_HOME: ~/.cache/bazel-repo
-        run: |
-          bazel --bazelrc=.aspect/bazelrc/ci.bazelrc \
-            --bazelrc=.github/workflows/ci.bazelrc \
-            --bazelrc=.aspect/bazelrc/bazel6.bazelrc \
-            test --config=local //...
-      - name: Build release artifacts
+          # NB: this variable is read by tools/release/copy_release_artifacts.sh
+          DEST: artifacts
         run: |
-          if [ -n "$(git status --porcelain)" ]; then
-            >&2 echo "ERROR: the git state is not clean, aborting build..."
-            exit 1
-          fi
-          rm -rf /tmp/aspect/release
-          bazel --bazelrc=.aspect/bazelrc/ci.bazelrc \
-            --bazelrc=.github/workflows/ci.bazelrc \
-            --bazelrc=.aspect/bazelrc/bazel6.bazelrc \
-            run --config=local //tools/release -- /tmp/aspect/release
+          rm -rf ${{ env.DEST }}
+          mkdir -p ${{ env.DEST }}
+          bazel --bazelrc=.github/workflows/ci.bazelrc \
+            run --config=release //tools/release:copy_release_artifacts
+      - uses: actions/upload-artifact@v4
+        with:
+          name: artifacts
+          path: artifacts/
+          retention-days: 1
+
+  release:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      # Fetch the built artifacts from build jobs above and extract into
+      # ${GITHUB_WORKSPACE}/artifacts/*
+      - uses: actions/download-artifact@v4
+
       - name: Prepare workspace snippet
-        run: .github/workflows/release_prep.sh ${{ env.GITHUB_REF_NAME }} > release_notes.txt
-      - name: Release
-        uses: softprops/action-gh-release@v1
+        run: .github/workflows/release_prep.sh > release_notes.txt
+
+      - uses: softprops/action-gh-release@v2
         with:
           # Use GH feature to populate the changelog automatically
           generate_release_notes: true
           files: |
-            /tmp/aspect/release/*
+            artifacts/*
             bazel-lib-*.tar.gz
           body_path: release_notes.txt
           fail_on_unmatched_files: true

.github/workflows/release_prep.sh

@@ -10,9 +10,37 @@ TAG=${GITHUB_REF_NAME}
 # with minimal differences in their code (e.g. strip_prefix remains the same)
 PREFIX="bazel-lib-${TAG:1}"
 ARCHIVE="bazel-lib-$TAG.tar.gz"
+ARCHIVE_TMP=$(mktemp)
 
 # NB: configuration for 'git archive' is in /.gitattributes
-git archive --format=tar --prefix=${PREFIX}/ ${TAG} | gzip >$ARCHIVE
+git archive --format=tar --prefix=${PREFIX}/ ${TAG} >$ARCHIVE_TMP
+
+############
+# Patch up the archive to have integrity hashes for built binaries that we downloaded in the GHA workflow.
+# Now that we've run `git archive` we are free to pollute the working directory.
+
+# Delete the placeholder file
+tar --file $ARCHIVE_TMP --delete ${PREFIX}/tools/integrity.bzl
+
+mkdir -p ${PREFIX}/tools
+cat >${PREFIX}/tools/integrity.bzl <<EOF
+"Generated during release by release_prep.sh, using integrity.jq"
+
+RELEASED_BINARY_INTEGRITY = $(
+  jq \
+    --from-file .github/workflows/integrity.jq \
+    --slurp \
+    --raw-input artifacts/*.sha256
+)
+EOF
+
+# Append that generated file back into the archive
+tar --file $ARCHIVE_TMP --append ${PREFIX}/tools/integrity.bzl
+
+# END patch up the archive
+############
+
+gzip <$ARCHIVE_TMP >$ARCHIVE
 SHA=$(shasum -a 256 $ARCHIVE | awk '{print $1}')
 
 cat <<EOF

e2e/coreutils/md5.txt

@@ -1 +1 @@
-160e389707f172f2d1333084fec4e650 test.bin
+160e389707f172f2d1333084fec4e650 test.bin

e2e/coreutils/sha1.txt

@@ -1 +1 @@
-b3dd96b0db6a12a3ae53899e8fbc04b749c980d6 test.bin
+b3dd96b0db6a12a3ae53899e8fbc04b749c980d6 test.bin

e2e/coreutils/sha256.txt

@@ -1 +1 @@
-3285d399e870df47749162ab2ebfd0b727e9142925ab5f5ad221c3a42a852b08 test.bin
+3285d399e870df47749162ab2ebfd0b727e9142925ab5f5ad221c3a42a852b08 test.bin

lib/private/copy_directory_toolchain.bzl

@@ -1,12 +1,7 @@
 "Setup copy_directory toolchain repositories and rules"
 
 # https://github.com/aspect-build/bazel-lib/releases
-#
-# The integrity hashes can be automatically fetched for the latest copy_directory release by running
-# `tools/copy_directory/mirror_release.sh`. To calculate for a specific release run
-# `tools/copy_directory/mirror_release.sh <release_version>`
-
-load("//tools:integrity.bzl", "COPY_DIRECTORY_INTEGRITY")
+load("//tools:integrity.bzl", "RELEASED_BINARY_INTEGRITY")
 load("//tools:version.bzl", "VERSION")
 
 # Platform names follow the platform naming convention in @aspect_bazel_lib//:lib/private/repo_utils.bzl
@@ -156,19 +151,19 @@ def _copy_directory_platform_repo_impl(rctx):
     is_windows = rctx.attr.platform.startswith("windows_")
     meta = COPY_DIRECTORY_PLATFORMS[rctx.attr.platform]
     release_platform = meta.release_platform if hasattr(meta, "release_platform") else rctx.attr.platform
+    release_file = "copy_directory-{}{}".format(release_platform, ".exe" if is_windows else "")
 
     # https://github.com/aspect-build/bazel-lib/releases/download/v1.19.0/copy_directory-linux_amd64
-    url = "https://github.com/aspect-build/bazel-lib/releases/download/v{0}/copy_directory-{1}{2}".format(
+    url = "https://github.com/aspect-build/bazel-lib/releases/download/v{}/{}".format(
         VERSION,
-        release_platform,
-        ".exe" if is_windows else "",
+        release_file,
     )
 
     rctx.download(
         url = url,
         output = "copy_directory.exe" if is_windows else "copy_directory",
         executable = True,
-        integrity = COPY_DIRECTORY_INTEGRITY[release_platform],
+        integrity = RELEASED_BINARY_INTEGRITY[release_file],
     )
     build_content = """# @generated by @aspect_bazel_lib//lib/private:copy_directory_toolchain.bzl
 load("@aspect_bazel_lib//lib/private:copy_directory_toolchain.bzl", "copy_directory_toolchain")

lib/private/copy_to_directory_toolchain.bzl

@@ -1,12 +1,7 @@
 "Setup copy_to_directory toolchain repositories and rules"
 
 # https://github.com/aspect-build/bazel-lib/releases
-#
-# The integrity hashes can be automatically fetched for the latest copy_to_directory release by running
-# `tools/copy_to_directory/mirror_release.sh`. To calculate for a specific release run
-# `tools/copy_to_directory/mirror_release.sh <release_version>`
-
-load("//tools:integrity.bzl", "COPY_TO_DIRECTORY_INTEGRITY")
+load("//tools:integrity.bzl", "RELEASED_BINARY_INTEGRITY")
 load("//tools:version.bzl", "VERSION")
 
 # Platform names follow the platform naming convention in @aspect_bazel_lib//:lib/private/repo_utils.bzl
@@ -156,19 +151,19 @@ def _copy_to_directory_platform_repo_impl(rctx):
     is_windows = rctx.attr.platform.startswith("windows_")
     meta = COPY_TO_DIRECTORY_PLATFORMS[rctx.attr.platform]
     release_platform = meta.release_platform if hasattr(meta, "release_platform") else rctx.attr.platform
+    release_file = "copy_to_directory-{}{}".format(release_platform, ".exe" if is_windows else "")
 
     # https://github.com/aspect-build/bazel-lib/releases/download/v1.19.0/copy_to_directory-linux_amd64
-    url = "https://github.com/aspect-build/bazel-lib/releases/download/v{0}/copy_to_directory-{1}{2}".format(
+    url = "https://github.com/aspect-build/bazel-lib/releases/download/v{}/{}".format(
         VERSION,
-        release_platform,
-        ".exe" if is_windows else "",
+        release_file,
     )
 
     rctx.download(
         url = url,
         output = "copy_to_directory.exe" if is_windows else "copy_to_directory",
         executable = True,
-        integrity = COPY_TO_DIRECTORY_INTEGRITY[release_platform],
+        integrity = RELEASED_BINARY_INTEGRITY[release_file],
     )
     build_content = """# @generated by @aspect_bazel_lib//lib/private:copy_to_directory_toolchain.bzl
 load("@aspect_bazel_lib//lib/private:copy_to_directory_toolchain.bzl", "copy_to_directory_toolchain")

lib/private/expand_template_toolchain.bzl

@@ -1,12 +1,7 @@
 "Setup expand_template toolchain repositories and rules"
 
 # https://github.com/aspect-build/bazel-lib/releases
-#
-# The integrity hashes can be automatically fetched for the latest expand_template release by running
-# `tools/expand_template/mirror_release.sh`. To calculate for a specific release run
-# `tools/expand_template/mirror_release.sh <release_version>`
-
-load("//tools:integrity.bzl", "EXPAND_TEMPLATE_INTEGRITY")
+load("//tools:integrity.bzl", "RELEASED_BINARY_INTEGRITY")
 load("//tools:version.bzl", "VERSION")
 
 # Platform names follow the platform naming convention in @aspect_bazel_lib//:lib/private/repo_utils.bzl
@@ -156,19 +151,19 @@ def _expand_template_platform_repo_impl(rctx):
     is_windows = rctx.attr.platform.startswith("windows_")
     meta = EXPAND_TEMPLATE_PLATFORMS[rctx.attr.platform]
     release_platform = meta.release_platform if hasattr(meta, "release_platform") else rctx.attr.platform
+    release_file = "expand_template-{}{}".format(release_platform, ".exe" if is_windows else "")
 
     # https://github.com/aspect-build/bazel-lib/releases/download/v1.19.0/expand_template-linux_amd64
-    url = "https://github.com/aspect-build/bazel-lib/releases/download/v{0}/expand_template-{1}{2}".format(
+    url = "https://github.com/aspect-build/bazel-lib/releases/download/v{}/{}".format(
         VERSION,
-        release_platform,
-        ".exe" if is_windows else "",
+        release_file,
     )
 
     rctx.download(
         url = url,
         output = "expand_template.exe" if is_windows else "expand_template",
         executable = True,
-        integrity = EXPAND_TEMPLATE_INTEGRITY[release_platform],
+        integrity = RELEASED_BINARY_INTEGRITY[release_file],
     )
     build_content = """# @generated by @aspect_bazel_lib//lib/private:expand_template_toolchain.bzl
 load("@aspect_bazel_lib//lib/private:expand_template_toolchain.bzl", "expand_template_toolchain")

lib/tests/run_binary_expansions/expansions_golden

@@ -1,16 +1,16 @@
-bazel-out/PLATFORM-opt/bin/lib/tests/run_binary_expansions/expansions_out
-bazel-out/PLATFORM-opt/bin/lib/tests/run_binary_expansions
+bazel-out/PLATFORM-fastbuild/bin/lib/tests/run_binary_expansions/expansions_out
+bazel-out/PLATFORM-fastbuild/bin/lib/tests/run_binary_expansions
 lib/tests/run_binary_expansions/src_1
 lib/tests/run_binary_expansions/src_1
-bazel-out/PLATFORM-opt/bin/lib/tests/run_binary_expansions/src_1
-bazel-out/PLATFORM-opt/bin/lib/tests/run_binary_expansions/src_1
+bazel-out/PLATFORM-fastbuild/bin/lib/tests/run_binary_expansions/src_1
+bazel-out/PLATFORM-fastbuild/bin/lib/tests/run_binary_expansions/src_1
 aspect_bazel_lib/lib/tests/run_binary_expansions/src_1
 aspect_bazel_lib/lib/tests/run_binary_expansions/src_1
-bazel-out/PLATFORM-opt/bin/lib/tests/run_binary_expansions/src_1
-bazel-out/PLATFORM-opt/bin/lib/tests/run_binary_expansions/src_1
-opt
-bazel-out/PLATFORM-opt/bin
-bazel-out/PLATFORM-opt/bin
+bazel-out/PLATFORM-fastbuild/bin/lib/tests/run_binary_expansions/src_1
+bazel-out/PLATFORM-fastbuild/bin/lib/tests/run_binary_expansions/src_1
+fastbuild
+bazel-out/PLATFORM-fastbuild/bin
+bazel-out/PLATFORM-fastbuild/bin
 PLATFORM
 lib/tests/run_binary_expansions/BUILD.bazel
 bazel-out/volatile-status.txt

lib/tests/run_binary_expansions/expansions_golden_bzlmod

@@ -1,16 +1,16 @@
-bazel-out/PLATFORM-opt/bin/lib/tests/run_binary_expansions/expansions_out
-bazel-out/PLATFORM-opt/bin/lib/tests/run_binary_expansions
+bazel-out/PLATFORM-fastbuild/bin/lib/tests/run_binary_expansions/expansions_out
+bazel-out/PLATFORM-fastbuild/bin/lib/tests/run_binary_expansions
 lib/tests/run_binary_expansions/src_1
 lib/tests/run_binary_expansions/src_1
-bazel-out/PLATFORM-opt/bin/lib/tests/run_binary_expansions/src_1
-bazel-out/PLATFORM-opt/bin/lib/tests/run_binary_expansions/src_1
+bazel-out/PLATFORM-fastbuild/bin/lib/tests/run_binary_expansions/src_1
+bazel-out/PLATFORM-fastbuild/bin/lib/tests/run_binary_expansions/src_1
 _main/lib/tests/run_binary_expansions/src_1
 _main/lib/tests/run_binary_expansions/src_1
-bazel-out/PLATFORM-opt/bin/lib/tests/run_binary_expansions/src_1
-bazel-out/PLATFORM-opt/bin/lib/tests/run_binary_expansions/src_1
-opt
-bazel-out/PLATFORM-opt/bin
-bazel-out/PLATFORM-opt/bin
+bazel-out/PLATFORM-fastbuild/bin/lib/tests/run_binary_expansions/src_1
+bazel-out/PLATFORM-fastbuild/bin/lib/tests/run_binary_expansions/src_1
+fastbuild
+bazel-out/PLATFORM-fastbuild/bin
+bazel-out/PLATFORM-fastbuild/bin
 PLATFORM
 lib/tests/run_binary_expansions/BUILD.bazel
 bazel-out/volatile-status.txt

tools/BUILD.bazel

@@ -1,19 +1,4 @@
 load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
-load("//lib:utils.bzl", "is_bazel_6_or_greater")
-load("//lib:write_source_files.bzl", "write_source_files")
-
-exports_files([
-    "create_release.sh",
-    "create_version.sh",
-])
-
-write_source_files(
-    name = "releases_versions_check_in",
-    files = {
-        "integrity.bzl": "//tools/release:release_versions",
-    },
-    tags = (["manual"] if not is_bazel_6_or_greater() else []),
-)
 
 bzl_library(
     name = "integrity",

tools/integrity.bzl

@@ -1,26 +1,10 @@
-"AUTO GENERATED. DO NOT EDIT"
+"""Release binary integrity hashes.
 
-COPY_DIRECTORY_INTEGRITY = {
-    "darwin_amd64": "sha256-EH6Qpf/IzIaGncigN+cMc2xCb0C3XuV8I4cUBtaZ7GE=",
-    "darwin_arm64": "sha256-DH2vl4k0MSyp+lnvfiiOu0ifc+tZSgJUIOFthSOMMvg=",
-    "freebsd_amd64": "sha256-ogXy1bGEMB4EnuF606H1Vi0h77B3xg+9rSnghDHyVEw=",
-    "linux_amd64": "sha256-QGFIoivc0z92barkw/JL4LbggV89nmCfsRkDK7fz4gY=",
-    "linux_arm64": "sha256-lSUkiCmhQaSxPNDaW8Ny+cipW1fcvNogX5Ex3zN1784=",
-    "windows_amd64": "sha256-ioAUxcSJhMRG7tghZRDH/WjATUEUjVyNN1Cs2BAozJs=",
-}
-COPY_TO_DIRECTORY_INTEGRITY = {
-    "darwin_amd64": "sha256-u2pIpD+qv/C58iLcJ0pfDs9U8kM2dIMMVW3YYTiRaBA=",
-    "darwin_arm64": "sha256-esM3e/Zez9ynrIhjgwq85ZEOd3KT9TZsDgsGuxIrNHw=",
-    "freebsd_amd64": "sha256-fJpbdVvTSwUfyGtngmaLeppFKdyw9BjFS0G/bYT8ZaY=",
-    "linux_amd64": "sha256-EoFMz8FEZIOSoUizTKnEQikrevmUwSw+JvPUidxAYa4=",
-    "linux_arm64": "sha256-+5u2Pz57OK64RDA4JDsvUnIkJUXkH4CfgCA3bx3vUPc=",
-    "windows_amd64": "sha256-nLybvOhMDWUw+2OyjfaFzmn08IwpO1tF80KwP2rrAPs=",
-}
-EXPAND_TEMPLATE_INTEGRITY = {
-    "darwin_amd64": "sha256-pu46U2pS+Sw54B1Bx0OBKvztzBBwnxLQp5fstbrx+To=",
-    "darwin_arm64": "sha256-wn39/0aGKGd6O6ZCZJnVIPwuScDieELUwrhRITHSPJU=",
-    "freebsd_amd64": "sha256-5q9RKZAyoxJiwD1dyjeAmo0g/sMVWM4m//DkmkJIZQo=",
-    "linux_amd64": "sha256-fuHVMGdTm9Ubfk5yufvGbV4g7d9dXUQqUu1kAiawmB4=",
-    "linux_arm64": "sha256-TLZIWcAB/YvXDM0RSGS/i7mO9ZadAiJ+uByX11uyJeI=",
-    "windows_amd64": "sha256-rhF8EkJ1y/3Hp/dKZwTJ3HtVPV+B6uqmtkhjUNFQRXA=",
+This file contents are entirely replaced during release publishing.
+The checked in content is only here to allow load() statements in the sources to resolve.
+"""
+
+RELEASED_BINARY_INTEGRITY = {
+    "copy_directory-darwin_amd64": "sha256-EH6Qpf/IzIaGncigN+cMc2xCb0C3XuV8I4cUBtaZ7GE=",
+    # ...etc
 }