feat(release-controller): various improvements

WORKSPACE.bazel

@@ -24,6 +24,18 @@ http_archive(
     url = "https://github.com/bazelbuild/rules_python/archive/c5c03b2477dd1ce0c06c9dc60bf816995f222bcf.zip",
 )
 
+http_file(
+    name = "bazelisk_darwin",
+    sha256 = "9a4b169038a63ebf60a9b4f367b449ab9b484c4ec7d1ef9f6b7a4196dfd50f33",
+    url = "https://github.com/bazelbuild/bazelisk/releases/download/v1.20.0/bazelisk-darwin",
+)
+
+http_file(
+    name = "bazelisk_linux",
+    sha256 = "d9af1fa808c0529753c3befda75123236a711d971d3485a390507122148773a3",
+    url = "https://github.com/bazelbuild/bazelisk/releases/download/v1.20.0/bazelisk-linux-amd64",
+)
+
 load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains")
 
 py_repositories()

release-controller/BUILD.bazel

@@ -1,6 +1,7 @@
 load("@python_deps//:requirements.bzl", "requirement")
 load("@rules_python//python:defs.bzl", "py_binary")
 load("@//tools/python:py_oci_image.bzl", "py_oci_image")
+load("@bazel_skylib//rules:native_binary.bzl", "native_binary")
 
 deps = [
     requirement("requests"),
@@ -42,13 +43,23 @@ py_binary(
     deps = deps,
 )
 
+native_binary(
+    name = "bazelisk",
+    src = select({
+        "@platforms//os:linux": "@bazelisk_linux//file",
+        "@platforms//os:macos": "@bazelisk_darwin//file",
+    }),
+    out = "bazelisk",
+)
+
 py_test(
     name = "pytest",
     srcs = ["pytest.py"],
-    data = glob(["*.py"]),
-    tags = ["no-sandbox"],
+    data = glob(["*.py"]) + [":bazelisk"],
     env = env,
+    tags = ["no-sandbox"],
     deps = deps + dev_deps,
+    env_inherit = ["HOME"],
 )
 
 py_oci_image(

release-controller/git_repo.py

@@ -2,13 +2,19 @@
 import pathlib
 import subprocess
 import tempfile
+import typing
 
 from dotenv import load_dotenv
 from release_index import Release
 from release_index import Version
 from util import version_name
 
 
+class FileChange(typing.TypedDict):
+    file_path: str
+    num_changes: int
+
+
 class Commit:
     """Class for representing a git commit."""
 
@@ -190,39 +196,53 @@ def distance(self, commit_a: str, commit_b: str) -> int:
             ).decode("utf-8")
         )
 
-    def get_commits_in_range(self, first_commit, last_commit):
-        """Get the commits in the range [first_commit, last_commit] from the IC repo."""
-        self.fetch()
-
-        def get_commits_info(git_commit_format):
-            return (
-                subprocess.check_output(
-                    [
-                        "git",
-                        "log",
-                        "--format={}".format(git_commit_format),
-                        "--no-merges",
-                        "{}..{}".format(first_commit, last_commit),
-                    ],
-                    cwd=self.dir,
-                    stderr=subprocess.DEVNULL,
-                )
-                .decode("utf-8")
-                .strip()
-                .split("\n")
+    def get_commits_info(
+        self,
+        git_commit_format: str,
+        first_commit: str,
+        last_commit: str,
+    ) -> list[str]:
+        """Get the info of commits in the range [first_commit, last_commit]."""
+        return (
+            subprocess.check_output(
+                [
+                    "git",
+                    "log",
+                    "--format={}".format(git_commit_format),
+                    "--no-merges",
+                    "{}..{}".format(first_commit, last_commit),
+                ],
+                cwd=self.dir,
+                stderr=subprocess.DEVNULL,
             )
+            .decode("utf-8")
+            .strip()
+            .split("\n")
+        )
 
-        commit_hash = get_commits_info("%h")
-        commit_message = get_commits_info("%s")
-        commiter = get_commits_info("%an")
-
-        return list(zip(commit_hash, commit_message, commiter))
+    def get_commit_info(
+        self,
+        git_commit_format: str,
+        first_commit: str,
+    ) -> str:
+        """Get the info of commit."""
+        return subprocess.check_output(
+            [
+                "git",
+                "show",
+                "-s",
+                "--format={}".format(git_commit_format),
+                first_commit,
+            ],
+            cwd=self.dir,
+            stderr=subprocess.DEVNULL,
+        ).decode("utf-8")
 
     def file(self, path: str) -> pathlib.Path:
         """Get the file for the given path."""
         return self.dir / path
 
-    def file_changes_for_commit(self, commit_hash):
+    def file_changes_for_commit(self, commit_hash) -> list[FileChange]:
         cmd = [
             "git",
             "diff",
@@ -250,13 +270,28 @@ def file_changes_for_commit(self, commit_hash):
 
             changes.append(
                 {
-                    "file_path": "/" + file_path,
+                    "file_path": file_path,
                     "num_changes": int(additions) + int(deletions),
                 }
             )
 
         return changes
 
+    def checkout(self, ref: str):
+        """Checkout the given ref."""
+        subprocess.check_call(
+            ["git", "reset", "--hard", f"origin/{self.main_branch}"],
+            cwd=self.dir,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+        subprocess.check_call(
+            ["git", "checkout", ref],
+            cwd=self.dir,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+
 
 # TODO: test
 def push_release_tags(repo: GitRepo, release: Release):

release-controller/google_docs.py

@@ -12,7 +12,7 @@
 from release_notes import prepare_release_notes
 
 md = markdown.Markdown(
-    extensions=["pymdownx.tilde"],
+    extensions=["pymdownx.tilde", "pymdownx.details"],
 )
 
 
@@ -94,12 +94,12 @@ def main():
         credentials_file=pathlib.Path(__file__).parent.resolve() / "credentials.json",
         release_notes_folder="1zOPwbYdOREhhLv-spRIRRMaFaAQlOVvF",
     )
-    version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"
+    version = "3d0b3f10417fc6708e8b5d844a0bac5e86f3e17d"
     gdoc = client.ensure(
-        release_tag="relase--2024-02-21_23-01-base",
+        release_tag="release-2024-08-02_01-30-base",
         release_commit=version,
-        base_release_commit="8d4b6898d878fa3db4028b316b78b469ed29f293",
-        base_release_tag="release--2024-02-14_23-01-base",
+        base_release_commit="2c0b76cfc7e596d5c4304cff5222a2619294c8c1",
+        base_release_tag="release-2024-07-25_21-03-base",
         tag_teams_on_create=False,
     )
     print(client.markdown_file(version))

release-controller/publish_notes.py

@@ -6,10 +6,56 @@
 from github import Auth
 from github import Github
 from github.Repository import Repository
+import markdown
 
 REPLICA_RELEASES_DIR = "replica-releases"
 
 
+def post_process_release_notes(release_notes: str) -> str:
+    """Process the release notes."""
+    lines = [
+        # add ticks around commit hash
+        re.sub(
+            r"(?<=\[)([a-h0-9]{9})(?=\])",
+            r"`\g<1>`",
+            # remove author
+            re.sub(r"(?<=^\* )(.*)author:[^|]+\| ?", r"\g<1>", line),
+        )
+        for line in release_notes.split("\n")
+    ]
+
+    changelog = "\n".join([line for line in lines if "~~" not in line])
+    excluded_lines = [line for line in lines if "~~" in line]
+    excluded_changes = "\n".join(
+        [
+            l
+            for l in [
+                re.sub(
+                    # remove whitespace after *
+                    r"(?<=^\* )\s+",
+                    "",
+                    # remove [AUTO-EXCLUDED]
+                    re.sub(r"\[AUTO-EXCLUDED[^]]*\]", "", line)
+                    # remove ~~
+                    .replace("~~", ""),
+                ).strip()
+                for line in excluded_lines
+            ]
+            if l.startswith("* [")
+        ]
+    )
+    if excluded_changes:
+        changelog += (
+            "\n<details>\n<summary>Other changes (either not directly modifying GuestOS or not relevant)</summary>\n"
+        )
+        md = markdown.Markdown(
+            extensions=["pymdownx.tilde"],
+        )
+        changelog += md.convert(excluded_changes)
+        changelog += "\n</details>\n"
+    return changelog
+
+
 class PublishNotesClient:
     """Publishes release notes on slack."""
 
@@ -46,21 +92,7 @@ def publish_if_ready(self, google_doc_markdownified, version: str):
             logging.info("didn't get markdown notes for %s, skipping", version)
             return
 
-        changelog = google_doc_markdownified
-        changelog = "\n".join(
-            [
-                # add ticks around commit hash
-                re.sub(
-                    r"(?<=^\* \[)([a-h0-9]{9})(?=\])",
-                    r"`\g<1>`",
-                    # remove author
-                    re.sub(r"(?<=^\* )author:[^|]+\| ", "", line),
-                )
-                for line in changelog.split("\n")
-                # remove crossed out lines (including reviewer checklist)
-                if "~~" not in line
-            ]
-        )
+        changelog = post_process_release_notes(google_doc_markdownified)
 
         release_notes_start = changelog.find("Release Notes")
         if release_notes_start == -1:

release-controller/reconciler.py

@@ -192,7 +192,7 @@ def reconcile(self):
                 continue
             rc_forum_topic = self.forum_client.get_or_create(rc)
             # update to create posts for any releases
-            rc_forum_topic.update(changelog=self.loader.changelog, proposal=self.state.version_proposal)
+            rc_forum_topic.update(changelog=self.loader.proposal_summary, proposal=self.state.version_proposal)
             for v_idx, v in enumerate(rc.versions):
                 logging.info("Updating version %s", v)
                 push_release_tags(self.ic_repo, rc)
@@ -210,7 +210,7 @@ def reconcile(self):
                 )
 
                 # returns a result only if changelog is published
-                changelog = self.loader.changelog(v.version)
+                changelog = self.loader.proposal_summary(v.version)
                 if changelog:
                     if not self.state.proposal_submitted(v.version):
                         unelect_versions = []
@@ -240,7 +240,7 @@ def reconcile(self):
                         self.state.save_proposal(v.version, versions_proposals[v.version])
 
             # update the forum posts in case the proposal was created
-            rc_forum_topic.update(changelog=self.loader.changelog, proposal=self.state.version_proposal)
+            rc_forum_topic.update(changelog=self.loader.proposal_summary, proposal=self.state.version_proposal)
 
 
 def place_proposal(ic_admin, changelog, version: str, forum_post_url: str, unelect_versions: list[str], dry_run=False):
@@ -329,7 +329,7 @@ def main():
 def oneoff():
     release_loader = GitReleaseLoader(f"https://github.com/{dre_repo}.git")
     version = "ac971e7b4c851b89b312bee812f6de542ed907c5"
-    changelog = release_loader.changelog(version)
+    changelog = release_loader.proposal_summary(version)
 
     ic_admin = IcAdmin("https://ic0.app", git_revision="5ba1412f9175d987661ae3c0d8dbd1ac3e092b7d")
     place_proposal(

release-controller/release_index_loader.py

@@ -5,6 +5,7 @@
 import tempfile
 
 from pydantic_yaml import parse_yaml_raw_as
+import re
 import release_index
 from git_repo import GitRepo
 from publish_notes import REPLICA_RELEASES_DIR
@@ -37,9 +38,15 @@ def index(self) -> release_index.Model:
     def changelog(self, version: str) -> str | None:
         version_changelog_path = self.release_index_dir / f"{REPLICA_RELEASES_DIR}/{version}.md"
         if version_changelog_path.exists():
-            return open(version_changelog_path, "r").read() + _verify_release_instructions(version)
+            return open(version_changelog_path, "r").read()
 
-        return None
+    def proposal_summary(self, version: str) -> str | None:
+        changelog = self.changelog(version)
+        if not changelog:
+            return None
+
+        # remove details html block from content for now until we can ensure it's going to fit into proposal size limits
+        return re.sub(r"\n<details>.*?</details>\n", "", changelog, flags=re.S) + _verify_release_instructions(version)
 
 
 class DevReleaseLoader(ReleaseLoader):
@@ -63,9 +70,9 @@ def index(self):
         self.git_repo.fetch()
         return super().index()
 
-    def changelog(self, version):
+    def proposal_summary(self, version):
         self.git_repo.fetch()
-        return super().changelog(version)
+        return super().proposal_summary(version)
 
 
 class StaticReleaseLoader(ReleaseLoader):