Merge generate_package_release_notes.py into release_notes.py

dev/releases/release_notes.py

@@ -21,14 +21,144 @@
 # Don't use this with versions like 4.13.0-beta1
 
 import json
+import gzip
+import os
+import requests
 import subprocess
 import sys
 
-from utils import error, notice
+from utils import error, notice, warning, is_existing_tag, download_with_sha256
 
-import typing
-from tempfile import TemporaryDirectory
-from typing import Any, Dict, List, Optional
+
+def usage(name: str) -> None:
+    print(f"Usage: `{name} NEWVERSION`")
+    sys.exit(1)
+
+
+def find_previous_version(version: str) -> str:
+    major, minor, patchlevel = version.split(".")
+    if major != "4":
+        error("unexpected GAP version, not starting with '4.'")
+    if patchlevel != "0":
+        patchlevel = int(patchlevel) - 1
+        return f"{major}.{minor}.{patchlevel}"
+    minor = int(minor) - 1
+    patchlevel = 0
+    while True:
+        v = f"{major}.{minor}.{patchlevel}"
+        if not is_existing_tag("v" + v):
+            break
+        patchlevel += 1
+    if patchlevel == 0:
+        error("could not determine previous version")
+    patchlevel -= 1
+    return f"{major}.{minor}.{patchlevel}"
+
+
+def package_infos_url(tag):
+    return f"https://github.com/gap-system/PackageDistro/releases/download/{tag}/package-infos.json.gz"
+
+
+def url_exists(url):
+    response = requests.get(url)
+    return response.status_code == 200
+
+
+def package_updates(relnotes_file, new_gap_version):
+    # create tmp directory
+    tmpdir = os.getcwd() + "/tmp"
+    notice(f"Files will be put in {tmpdir}")
+    try:
+        os.mkdir(tmpdir)
+    except FileExistsError:
+        pass
+
+    old_gap_version = find_previous_version(new_gap_version)
+    notice(
+        f"generating package release notes for {old_gap_version} -> {new_gap_version}"
+    )
+
+    oldtag = "v" + old_gap_version
+    newtag = "v" + new_gap_version
+    if not url_exists(package_infos_url(newtag)):
+        warning(f"no package infos found for {newtag}, switching to latest")
+        newtag = "latest"
+
+    # download package metadata
+    old_json_file = f"{tmpdir}/package-infos-{oldtag}.json.gz"
+    new_json_file = f"{tmpdir}/package-infos-{newtag}.json.gz"
+
+    download_with_sha256(package_infos_url(oldtag), old_json_file)
+    download_with_sha256(package_infos_url(newtag), new_json_file)
+
+    # parse package metadata
+    with gzip.open(old_json_file, "r") as f:
+        old_json = json.load(f)
+
+    with gzip.open(new_json_file, "r") as f:
+        new_json = json.load(f)
+
+    relnotes_file.write("### Package distribution\n\n")
+
+    #
+    # Detect new packages
+    #
+    added = new_json.keys() - old_json.keys()
+    if len(added) > 0:
+        relnotes_file.write("#### New packages redistributed with GAP\n\n")
+        for p in sorted(added):
+            pkg = new_json[p]
+            name = pkg["PackageName"]
+            home = pkg["PackageWWWHome"]
+            desc = pkg["Subtitle"]
+            vers = pkg["Version"]
+            authors = [
+                x["FirstNames"] + " " + x["LastName"]
+                for x in pkg["Persons"]
+                if x["IsAuthor"]
+            ]
+            authors = ", ".join(authors)
+            relnotes_file.write(
+                f"- [**{name}**]({home}) {vers}: {desc}, by {authors}\n"
+            )
+        relnotes_file.write("\n")
+
+    #
+    # Detect new packages
+    #
+    removed = old_json.keys() - new_json.keys()
+    if len(removed) > 0:
+        relnotes_file.write("#### Packages no longer redistributed with GAP\n\n")
+        for p in sorted(removed):
+            name = old_json[p]["PackageName"]
+            relnotes_file.write(f"- **{name}**: TODO\n")
+        relnotes_file.write("\n")
+
+    #
+    # Detect new packages
+    #
+    updated = new_json.keys() & old_json.keys()
+    updated = [p for p in updated if old_json[p]["Version"] != new_json[p]["Version"]]
+    if len(updated) > 0:
+        relnotes_file.write(
+            f"""
+#### Updated packages redistributed with GAP
+
+The GAP {new_gap_version} distribution contains {len(new_json)} packages, of which {len(updated)} have been
+updated since GAP {old_gap_version}. The full list of updated packages is given below:
+
+""".lstrip()
+        )
+        for p in sorted(updated):
+            old = old_json[p]
+            new = new_json[p]
+            name = new["PackageName"]
+            home = new["PackageWWWHome"]
+            oldversion = old["Version"]
+            newversion = new["Version"]
+            relnotes_file.write(
+                f"- [**{name}**]({home}): {oldversion} -> {newversion}\n"
+            )
 
 
 # the following is a list of pairs [LABEL, DESCRIPTION]; the first entry is the name of a GitHub label
@@ -59,11 +189,6 @@
 ]
 
 
-def usage(name: str) -> None:
-    print(f"Usage: `{name} NEWVERSION`")
-    sys.exit(1)
-
-
 def get_tag_date(tag: str) -> str:
     # TODO: validate the tag exists
     res = subprocess.run(
@@ -79,7 +204,7 @@ def get_tag_date(tag: str) -> str:
 
 def get_pr_list(date: str, extra: str) -> str:
     query = f'merged:>={date} -label:"release notes: not needed" -label:"release notes: added" base:master {extra}'
-    print("query = ", query)
+    print("query: ", query)
     res = subprocess.run(
         [
             "gh",
@@ -116,8 +241,9 @@ def changes_overview(prs, startdate, new_version):
     """Writes files with information for release notes."""
 
     # Could also introduce some consistency checks here for wrong combinations of labels
-
-    relnotes_file = open("releasenotes_" + new_version + ".md", "w")
+    filename = "releasenotes_" + new_version + ".md"
+    notice("Writing release notes into file "+filename)
+    relnotes_file = open(filename, "w")
     prs_with_use_title = [pr for pr in prs if has_label(pr, "release notes: use title")]
 
     # Write out all PRs with 'use title'
@@ -152,8 +278,13 @@ def changes_overview(prs, startdate, new_version):
         for pr in prs_with_use_title:
             relnotes_file.write(pr_to_md(pr))
         relnotes_file.write("\n")
+
+    package_updates(relnotes_file, new_version)
+
     relnotes_file.close()
 
+    notice("Release notes were written into file "+filename)
+
     unsorted_file = open("unsorted_PRs_" + new_version + ".md", "w")
 
     # Report PRs that have to be updated before inclusion into release notes.
@@ -220,25 +351,12 @@ def main(new_version: str) -> None:
     prs = get_pr_list(startdate, extra)
     # print(json.dumps(prs, sort_keys=True, indent=4))
 
-    # further filtering n
-    # prs = filter_prs(prs, rel_type)
-
     changes_overview(prs, startdate, new_version)
 
-    print("======= END =======")
-
-
-# TODO: allow specifying tags instead and download the json files from there
-# TODO: "guess" the tag of the previous/old version so it can be completely omitted
-# TODO: if the new package list is omitted, download the one from the PackageDistro?
-# TODO: integrate this script into generate_release_notes.py
-
 
 if __name__ == "__main__":
     # the argument is the new version
-    # TODO ... and for now also the old version???
     if len(sys.argv) != 2:
         usage(sys.argv[0])
 
     main(sys.argv[1])
-    # TODO: integrate generate_package_release_notes.py