Skip to content

Commit

Permalink
Cleanup python scripts in dev/release
Browse files Browse the repository at this point in the history
  • Loading branch information
james-d-mitchell committed Jan 31, 2024
1 parent 2d6dd7c commit 96e0ad4
Show file tree
Hide file tree
Showing 8 changed files with 360 additions and 223 deletions.
5 changes: 3 additions & 2 deletions dev/releases/create_stable_branch.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,12 @@
# TODO: implement parts of the steps described in
# <https://github.com/gap-system/gap-distribution/blob/master/DistributionUpdate/STABLE_BRANCH_CHECKLIST.md>

import subprocess
import sys

from utils import error, notice, patchfile
import utils

import subprocess
import sys

# Insist on Python >= 3.6 for f-strings and other goodies
if sys.version_info < (3, 6):
Expand Down
29 changes: 22 additions & 7 deletions dev/releases/generate_package_release_notes.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,17 @@
import json
import gzip

from utils import *
from utils import download_with_sha256


def usage():
print("Usage: `./generate_package_release_notes.py OLD_GAP_VERSION NEW_GAP_VERSION`")
print(
"Usage: `./generate_package_release_notes.py OLD_GAP_VERSION NEW_GAP_VERSION`"
)
sys.exit(1)

def main(old_gap_version, new_gap_version):

def main(old_gap_version, new_gap_version):
# create tmp directory
tmpdir = os.getcwd() + "/tmp"
notice(f"Files will be put in {tmpdir}")
Expand All @@ -40,9 +43,15 @@ def main(old_gap_version, new_gap_version):

# download package metadata
old_json_file = f"{tmpdir}/package-infos-{old_gap_version}.json.gz"
download_with_sha256(f"https://github.com/gap-system/PackageDistro/releases/download/v{old_gap_version}/package-infos.json.gz", old_json_file)
download_with_sha256(
f"https://github.com/gap-system/PackageDistro/releases/download/v{old_gap_version}/package-infos.json.gz",
old_json_file,
)
new_json_file = f"{tmpdir}/package-infos-{new_gap_version}.json.gz"
download_with_sha256(f"https://github.com/gap-system/PackageDistro/releases/download/v{new_gap_version}/package-infos.json.gz", new_json_file)
download_with_sha256(
f"https://github.com/gap-system/PackageDistro/releases/download/v{new_gap_version}/package-infos.json.gz",
new_json_file,
)

# parse package metadata
with gzip.open(old_json_file, "r") as f:
Expand All @@ -67,7 +76,11 @@ def main(old_gap_version, new_gap_version):
home = pkg["PackageWWWHome"]
desc = pkg["Subtitle"]
vers = pkg["Version"]
authors = [x["FirstNames"]+" "+x["LastName"] for x in pkg["Persons"] if x["IsAuthor"]]
authors = [
x["FirstNames"] + " " + x["LastName"]
for x in pkg["Persons"]
if x["IsAuthor"]
]
authors = ", ".join(authors)
print(f"- [**{name}**]({home}) {vers}: {desc}, by {authors}")
print()
Expand All @@ -90,7 +103,8 @@ def main(old_gap_version, new_gap_version):
updated = new_json.keys() & old_json.keys()
updated = [p for p in updated if old_json[p]["Version"] != new_json[p]["Version"]]
if len(updated) > 0:
print(f"""
print(
f"""
#### Updated packages redistributed with GAP
The GAP {new_gap_version} distribution contains {len(new_json)} packages, of which {len(updated)} have been
Expand All @@ -105,6 +119,7 @@ def main(old_gap_version, new_gap_version):
newversion = new["Version"]
print(f"- [**{name}**]({home}): {oldversion} -> {newversion}")


if __name__ == "__main__":
if len(sys.argv) != 3:
usage()
Expand Down
137 changes: 91 additions & 46 deletions dev/releases/make_archives.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,25 +9,39 @@
## SPDX-License-Identifier: GPL-2.0-or-later
##
## This script create the archives that form a GA{} release.
##
##
## The version of the gap release is taken from the Makefile variable
## GAP_BUILD_VERSION.
##
from utils import *
##

import glob
import grp
import gzip
import json
import os
import pwd
import re
import shutil
import subprocess
import sys
import tarfile

from utils import (
download_with_sha256,
error,
get_makefile_var,
notice,
patchfile,
run_with_log,
safe_git_fetch_tags,
verify_command_available,
verify_git_clean,
verify_git_repo,
working_directory,
)

# Insist on Python >= 3.6 for f-strings and other goodies
if sys.version_info < (3,6):
if sys.version_info < (3, 6):
error("Python 3.6 or newer is required")

notice("Checking prerequisites")
Expand All @@ -54,70 +68,91 @@
gapversion = get_makefile_var("GAP_BUILD_VERSION")
except:
error("make sure GAP has been compiled via './configure && make'")

notice(f"Detected GAP version {gapversion}")

if re.fullmatch( r"[1-9]+\.[0-9]+\.[0-9]+", gapversion) != None:
notice(f"--- THIS LOOKS LIKE A RELEASE ---")
if re.fullmatch(r"[1-9]+\.[0-9]+\.[0-9]+", gapversion) != None:
notice("--- THIS LOOKS LIKE A RELEASE ---")
pkg_tag = f"v{gapversion}"
else:
notice(f"--- THIS LOOKS LIKE A NIGHTLY BUILD ---")
notice("--- THIS LOOKS LIKE A NIGHTLY BUILD ---")
pkg_tag = "latest"


# extract commit_date with format YYYY-MM-DD
# TODO: is that really what we want? Or should it be the date this
# script was run? Or for releases, perhaps use the TaggerDate of the
# release tag (but then we need to find and process that tag)
commit_date = subprocess.run(["git", "show", "-s", "--format=%as"],
check=True, capture_output=True, text=True)
commit_date = subprocess.run(
["git", "show", "-s", "--format=%as"], check=True, capture_output=True, text=True
)
commit_date = commit_date.stdout.strip()
commit_year = commit_date[0:4]

# derive tarball names
basename = f"gap-{gapversion}"
all_packages = f"packages-v{gapversion}" # only the pkg dir
all_packages = f"packages-v{gapversion}" # only the pkg dir
all_packages_tarball = f"{all_packages}.tar.gz"
req_packages = f"packages-required-v{gapversion}" # a subset of the above
req_packages = f"packages-required-v{gapversion}" # a subset of the above
req_packages_tarball = f"{req_packages}.tar.gz"

PKG_BOOTSTRAP_URL = f"https://github.com/gap-system/PackageDistro/releases/download/{pkg_tag}/"
PKG_BOOTSTRAP_URL = (
f"https://github.com/gap-system/PackageDistro/releases/download/{pkg_tag}/"
)
PKG_MINIMAL = "packages-required.tar.gz"
PKG_FULL = "packages.tar.gz"

# Exporting repository content into tmp
notice("Exporting repository content via `git archive`")
rawbasename = "gap-raw"
rawgap_tarfile = f"{tmpdir}/{rawbasename}.tar"
subprocess.run(["git", "archive",
f"--prefix={basename}/",
f"--output={rawgap_tarfile}",
"HEAD"], check=True)
subprocess.run(
["git", "archive", f"--prefix={basename}/", f"--output={rawgap_tarfile}", "HEAD"],
check=True,
)

notice("Extracting exported content")
shutil.rmtree(basename, ignore_errors=True) # remove any leftovers
shutil.rmtree(basename, ignore_errors=True) # remove any leftovers
with tarfile.open(rawgap_tarfile) as tar:
tar.extractall(path=tmpdir)
os.remove(rawgap_tarfile)

notice("Processing exported content")
manifest_list = [] # collect names of assets to be uploaded to GitHub release
manifest_list = [] # collect names of assets to be uploaded to GitHub release

# download package distribution
notice("Downloading package distribution") # ... outside of the directory we just created
download_with_sha256(PKG_BOOTSTRAP_URL+"package-infos.json.gz", tmpdir+"/"+"package-infos.json.gz")
# outside of the directory we just created
notice("Downloading package distribution")
download_with_sha256(
PKG_BOOTSTRAP_URL + "package-infos.json.gz", tmpdir + "/" + "package-infos.json.gz"
)
manifest_list.append("package-infos.json.gz")
download_with_sha256(PKG_BOOTSTRAP_URL+PKG_MINIMAL, tmpdir+"/"+req_packages_tarball)
download_with_sha256(
PKG_BOOTSTRAP_URL + PKG_MINIMAL, tmpdir + "/" + req_packages_tarball
)
manifest_list.append(req_packages_tarball)
download_with_sha256(PKG_BOOTSTRAP_URL+PKG_FULL, tmpdir+"/"+all_packages_tarball)
download_with_sha256(PKG_BOOTSTRAP_URL + PKG_FULL, tmpdir + "/" + all_packages_tarball)
manifest_list.append(all_packages_tarball)

with working_directory(tmpdir + "/" + basename):
# This sets the version, release day and year of the release we are
# creating.
notice("Patching configure.ac")
patchfile("configure.ac", r"m4_define\(\[gap_version\],[^\n]+", r"m4_define([gap_version], ["+gapversion+"])")
patchfile("configure.ac", r"m4_define\(\[gap_releaseday\],[^\n]+", r"m4_define([gap_releaseday], ["+commit_date+"])")
patchfile("configure.ac", r"m4_define\(\[gap_releaseyear\],[^\n]+", r"m4_define([gap_releaseyear], ["+commit_year+"])")
patchfile(
"configure.ac",
r"m4_define\(\[gap_version\],[^\n]+",
r"m4_define([gap_version], [" + gapversion + "])",
)
patchfile(
"configure.ac",
r"m4_define\(\[gap_releaseday\],[^\n]+",
r"m4_define([gap_releaseday], [" + commit_date + "])",
)
patchfile(
"configure.ac",
r"m4_define\(\[gap_releaseyear\],[^\n]+",
r"m4_define([gap_releaseyear], [" + commit_year + "])",
)

# Building GAP
notice("Running autogen.sh")
Expand Down Expand Up @@ -153,36 +188,45 @@
shutil.rmtree("hpcgap-build")

notice("Extracting package tarballs")
with tarfile.open(tmpdir+"/"+all_packages_tarball) as tar:
with tarfile.open(tmpdir + "/" + all_packages_tarball) as tar:
tar.extractall(path="pkg")
# for some reason pkg sometimes ends up with permission 0700 so
# we make sure to fix that here
subprocess.run(["chmod", "0755", "pkg"], check=True)
# ensure all files are at readable by everyone
subprocess.run(["chmod", "-R", "a+r", "."], check=True)

with tarfile.open(tmpdir+"/"+req_packages_tarball) as tar:
tar.extractall(path=tmpdir+"/"+req_packages)
with tarfile.open(tmpdir + "/" + req_packages_tarball) as tar:
tar.extractall(path=tmpdir + "/" + req_packages)

notice("Building GAP's manuals")
run_with_log(["make", "doc"], "gapdoc", "building the manuals")

# Now we create the help-links.json file. We build
# the json package, create the files, then clean up the package again.
notice("Compiling json package")
path_to_json_package = glob.glob(f'{tmpdir}/{basename}/pkg/json*')[0]
path_to_json_package = glob.glob(f"{tmpdir}/{basename}/pkg/json*")[0]
with working_directory(path_to_json_package):
subprocess.run(["./configure"], check=True)
subprocess.run(["make"], check=True)

notice(f"Constructing help-links JSON file")
notice("Constructing help-links JSON file")
json_output = subprocess.run(
["./gap", "-r", "--quiet", "--quitonbreak", f"dev/releases/HelpLinks-to-JSON.g"],
check=True, capture_output=True, text=True)
[
"./gap",
"-r",
"--quiet",
"--quitonbreak",
"dev/releases/HelpLinks-to-JSON.g",
],
check=True,
capture_output=True,
text=True,
)
formatted_json = json.dumps(json.loads(json_output.stdout), indent=2)
with working_directory(tmpdir):
with gzip.open("help-links.json.gz", 'wb') as file:
file.write(formatted_json.encode('utf-8'))
with gzip.open("help-links.json.gz", "wb") as file:
file.write(formatted_json.encode("utf-8"))
manifest_list.append("help-links.json.gz")

notice("Cleaning up the json package")
Expand All @@ -193,11 +237,11 @@

notice("Removing unwanted version-controlled files")
badfiles = [
".codecov.yml",
".ctags",
".gitattributes",
".gitignore",
".mailmap",
".codecov.yml",
".ctags",
".gitattributes",
".gitignore",
".mailmap",
]

shutil.rmtree("benchmark")
Expand Down Expand Up @@ -226,29 +270,30 @@ def make_and_record_archive(name, compression, root_dir, base_dir):
else:
error(f"unknown compression type {compression} (not gztar or zip)")

filename = f"{name}{ext}"
notice(f"Creating {filename}")
fname = name + ext
notice(f"Creating {fname}")
owner = pwd.getpwuid(0).pw_name
group = grp.getgrgid(0).gr_name
shutil.make_archive(name, compression, root_dir, base_dir, owner = owner, group = group)
manifest_list.append(filename)
shutil.make_archive(name, compression, root_dir, base_dir, owner=owner, group=group)
manifest_list.append(fname)


# Create the remaining archives
notice("Creating remaining GAP and package archives")
with working_directory(tmpdir):
make_and_record_archive(basename, "gztar", ".", basename)
make_and_record_archive(basename, "zip", ".", basename)
make_and_record_archive(basename, "zip", ".", basename)
make_and_record_archive(all_packages, "zip", basename, "pkg")
make_and_record_archive(req_packages, "zip", ".", req_packages)
notice("Removing packages to facilitate creating the GAP core archives")
shutil.rmtree(basename + "/pkg")
make_and_record_archive(basename + "-core", "gztar", ".", basename)
make_and_record_archive(basename + "-core", "zip", ".", basename)
make_and_record_archive(basename + "-core", "zip", ".", basename)

# If you create additional archives, make sure to add them to manifest_list!
manifest_filename = "MANIFEST"
notice(f"Creating the manifest, with name {manifest_filename}")
with open(manifest_filename, 'w') as manifest:
with open(manifest_filename, "w", encoding="utf-8") as manifest:
for filename in manifest_list:
manifest.write(f"{filename}\n")

Expand Down
Loading

0 comments on commit 96e0ad4

Please sign in to comment.