From f488610c841c9e2676e4decc60094a25446743ec Mon Sep 17 00:00:00 2001
From: Alexander Konovalov <alexk@mcs.st-andrews.ac.uk>
Date: Fri, 19 Feb 2021 14:43:49 +0000
Subject: [PATCH 01/23] Initial Python script to get an overview of changes
 with PyGithub

---
 dev/release-notes.py | 57 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100755 dev/release-notes.py

diff --git a/dev/release-notes.py b/dev/release-notes.py
new file mode 100755
index 0000000000..80f6888904
--- /dev/null
+++ b/dev/release-notes.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+#
+# Usage ./release-notes.py
+#
+
+import sys
+from github import Github
+from datetime import datetime
+
+def get_prs(repo,startdate):
+    prs = {}
+    all_pulls = repo.get_pulls(state='closed', sort='created', direction='desc', base='master')
+    for pr in all_pulls:
+        if pr.merged:
+            print(pr.number, pr.title)
+            if pr.closed_at > datetime.fromisoformat(startdate):
+                labs = [lab.name for lab in list(pr.get_labels())]
+                if 'release notes: added' in labs:
+                    prs[pr.number] = { "title" : pr.title, 
+                                       "closed_at" : pr.closed_at.isoformat(), 
+                                       "labels" : [lab for lab in labs if lab.startswith('kind') or lab.startswith('topic')] }
+                    if len(prs)>10: # for quick testing
+                        break
+    return prs;
+
+# TODO - print markdown to a file, on screen for now
+def changes_overview(prs,startdate):
+    print("--------------------------------------")
+    print("## Changes since", startdate)
+    for k in prs:
+        print(k,prs[k]["title"], prs[k]["labels"] )
+    print("--------------------------------------")
+
+def main(startdate):
+
+    # Authentication and checking current API capacity
+    # TODO: for now this will do, use Sergio's code later
+    with open('/Users/alexk/.github_shell_token', 'r') as f:
+        accessToken=f.read().replace('\n', '')
+    g=Github(accessToken)
+
+    orgName = "gap-system"
+    repoName = "gap"
+    repo = g.get_repo( orgName + "/" + repoName)
+    
+    print("Current GitHub API capacity", g.rate_limiting)
+    prs = get_prs(repo,startdate)
+    changes_overview(prs,startdate)
+    print("Remaining GitHub API capacity", g.rate_limiting)
+
+
+if __name__ == "__main__":
+    print("script name is", sys.argv[0])
+    if (len(sys.argv) != 2): # the argument is the start date in ISO 8601
+        usage()              # to be defined
+       	sys.exit(0)          # exit with return value of 0
+    main(sys.argv[1])

From ba926faf402c2356cf873b17bcabde582c79b47a Mon Sep 17 00:00:00 2001
From: danielrademacher <33546089+danielrademacher@users.noreply.github.com>
Date: Fri, 19 Feb 2021 16:58:17 +0100
Subject: [PATCH 02/23] Pretty-printing of changes overview

---
 dev/release-notes.py | 126 +++++++++++++++++++++++++++++++++++++++----
 1 file changed, 115 insertions(+), 11 deletions(-)

diff --git a/dev/release-notes.py b/dev/release-notes.py
index 80f6888904..a8a845cea5 100755
--- a/dev/release-notes.py
+++ b/dev/release-notes.py
@@ -12,24 +12,128 @@ def get_prs(repo,startdate):
     all_pulls = repo.get_pulls(state='closed', sort='created', direction='desc', base='master')
     for pr in all_pulls:
         if pr.merged:
-            print(pr.number, pr.title)
             if pr.closed_at > datetime.fromisoformat(startdate):
                 labs = [lab.name for lab in list(pr.get_labels())]
-                if 'release notes: added' in labs:
-                    prs[pr.number] = { "title" : pr.title, 
-                                       "closed_at" : pr.closed_at.isoformat(), 
-                                       "labels" : [lab for lab in labs if lab.startswith('kind') or lab.startswith('topic')] }
-                    if len(prs)>10: # for quick testing
-                        break
+                prs[pr.number] = { "title" : pr.title,
+                                    "closed_at" : pr.closed_at.isoformat(),
+                                    "labels" : labs }
+                # How long do we have to run this?
+                if len(prs)>423: # for quick testing
+                    break
     return prs;
 
 # TODO - print markdown to a file, on screen for now
 def changes_overview(prs,startdate):
-    print("--------------------------------------")
+    #print("--------------------------------------")
+    #print("## Changes since", startdate)
+    #for k in prs:
+    #    print(k,prs[k]["title"], prs[k]["labels"] )
+    #print("--------------------------------------")
     print("## Changes since", startdate)
+    f = open("releasenotes.md", "a")
+    f2 = open("remainingPR.md", "a")
+    f3 = open("releasenotes.json", "a")
+    jsondict = prs.copy()
+
+    prioritylist = ["kind: bug: wrong result", "kind: bug: crash", "kind: bug: unexpected error", "kind: bug", "kind: enhancement", "kind: new feature", "kind: performance", "topic:libgap", "topic: julia", "topic: documentation", "topic: packages"]
+
+    f.write("Release Notes \n\n\n\n")
+    f.write("Category " + "release notes: highlight" + "\n")
+    removelist = []
+    for k in prs:
+        # The format of an entry of list is: ['title of PR', 'Link' (Alternative the PR number can be used), [ list of labels ] ]
+        if "release notes: highlight" in prs[k]["labels"]:
+            f.write("- [#")
+            issuenumber = str(k)
+            f.write(issuenumber)
+            f.write("](")
+            f.write("https://github.com/gap-system/gap/pull/" + str(k))
+            f.write(") ")
+            f.write(prs[k]["title"])
+            f.write("\n")
+            removelist.append(k)
+    for item in removelist:
+        del prs[item]
+    f.write("\n\n\n")
+
+
+    removelist = []
+    for k in prs:
+        if "release notes: not needed" in prs[k]["labels"]:
+            removelist.append(k)
+    for item in removelist:
+        del prs[item]
+        del jsondict[item]
+
+
+    f2.write("Category " + "release notes: to be added" + "\n")
+    removelist = []
     for k in prs:
-        print(k,prs[k]["title"], prs[k]["labels"] )
-    print("--------------------------------------")
+        if "release notes: to be added" in prs[k]["labels"]:
+            f2.write("- [#")
+            issuenumber = str(k)
+            f2.write(issuenumber)
+            f2.write("](")
+            f2.write("https://github.com/gap-system/gap/pull/" + str(k))
+            f2.write(") ")
+            f2.write(prs[k]["title"])
+            f2.write("\n")
+            removelist.append(k)
+    for item in removelist:
+        del prs[item]
+    f2.write("\n\n\n")
+
+
+    f2.write("Uncategorized PR" + "\n")
+    removelist = []
+    for k in prs:
+        #if not "release notes: use title" in item[2]:
+        if not "release notes: added" in prs[k]["labels"]:
+            f2.write("- [#")
+            issuenumber = str(k)
+            f2.write(issuenumber)
+            f2.write("](")
+            f2.write("https://github.com/gap-system/gap/pull/" + str(k))
+            f2.write(") ")
+            f2.write(prs[k]["title"])
+            f2.write("\n")
+            removelist.append(k)
+    for item in removelist:
+        del prs[item]
+    f2.close()
+    
+
+    for priorityobject in prioritylist:
+        f.write("Category " + priorityobject + "\n")
+        removelist = []
+        for k in prs:
+            if priorityobject in prs[k]["labels"]:
+                f.write("- [#")
+                issuenumber = str(k)
+                f.write(issuenumber)
+                f.write("](")
+                f.write("https://github.com/gap-system/gap/pull/" + str(k))
+                f.write(") ")
+                f.write(prs[k]["title"])
+                f.write("\n")
+                removelist.append(k)
+        for item in removelist:
+            del prs[item]
+        f.write("\n\n\n")
+    f.close()
+
+    f3.write("[")
+    jsonlist = []
+    for k in jsondict:
+        temp = []
+        temp.append(str(jsondict[k]["title"]))
+        temp.append(str(k))
+        temp.append(jsondict[k]["labels"])
+        jsonlist.append(temp)
+    for item in jsonlist:
+        f3.write("%s\n" % item)
+    f3.write("]")
+    f3.close
 
 def main(startdate):
 
@@ -48,7 +152,7 @@ def main(startdate):
     changes_overview(prs,startdate)
     print("Remaining GitHub API capacity", g.rate_limiting)
 
-
+    
 if __name__ == "__main__":
     print("script name is", sys.argv[0])
     if (len(sys.argv) != 2): # the argument is the start date in ISO 8601

From 4d43a966f84c7463fd4931db23cf2d5b16f95d1a Mon Sep 17 00:00:00 2001
From: Alexander Konovalov <alexk@mcs.st-andrews.ac.uk>
Date: Fri, 19 Feb 2021 19:41:54 +0000
Subject: [PATCH 03/23] Reset output files; Also documenting, TODOs, formatting

---
 dev/release-notes.py | 51 +++++++++++++++++++++++++++++++-------------
 1 file changed, 36 insertions(+), 15 deletions(-)

diff --git a/dev/release-notes.py b/dev/release-notes.py
index a8a845cea5..1b5dd58027 100755
--- a/dev/release-notes.py
+++ b/dev/release-notes.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Usage ./release-notes.py
+# Usage ./release-notes.py YYYY-MM-DD
 #
 
 import sys
@@ -8,34 +8,49 @@
 from datetime import datetime
 
 def get_prs(repo,startdate):
+    """Returns a dictionary of PRs matching selection criteria."""
     prs = {}
     all_pulls = repo.get_pulls(state='closed', sort='created', direction='desc', base='master')
+    # We need to run this over the whole list of PRs. Sorting by creation date descending
+    # is not really helping - could be that some very old PRs are being merged.
     for pr in all_pulls:
+        print(pr.number, end=' ')
+        # flush stdout immediately, to see progress indicator
+        sys.stdout.flush()
         if pr.merged:
             if pr.closed_at > datetime.fromisoformat(startdate):
                 labs = [lab.name for lab in list(pr.get_labels())]
                 prs[pr.number] = { "title" : pr.title,
                                     "closed_at" : pr.closed_at.isoformat(),
                                     "labels" : labs }
-                # How long do we have to run this?
-                if len(prs)>423: # for quick testing
+                if len(prs)>50: # for quick testing
                     break
-    return prs;
+    print("\n")
+    return prs
 
-# TODO - print markdown to a file, on screen for now
 def changes_overview(prs,startdate):
-    #print("--------------------------------------")
-    #print("## Changes since", startdate)
-    #for k in prs:
-    #    print(k,prs[k]["title"], prs[k]["labels"] )
-    #print("--------------------------------------")
+    """Writes files with information for release notes."""
+
+    #TODO: using cached data, check that the starting date is the same
+    # Also save the date when cache was saved (warning if old?)
     print("## Changes since", startdate)
-    f = open("releasenotes.md", "a")
-    f2 = open("remainingPR.md", "a")
-    f3 = open("releasenotes.json", "a")
+
+    # Opening files with "w" resets them
+    f = open("releasenotes.md", "w")
+    f2 = open("remainingPR.md", "w")
+    f3 = open("releasenotes.json", "w")
     jsondict = prs.copy()
 
-    prioritylist = ["kind: bug: wrong result", "kind: bug: crash", "kind: bug: unexpected error", "kind: bug", "kind: enhancement", "kind: new feature", "kind: performance", "topic:libgap", "topic: julia", "topic: documentation", "topic: packages"]
+    prioritylist = ["kind: bug: wrong result", 
+                    "kind: bug: crash", 
+                    "kind: bug: unexpected error", 
+                    "kind: bug", "kind: enhancement", 
+                    "kind: new feature", 
+                    "kind: performance", 
+                    "topic:libgap", 
+                    "topic: julia", 
+                    "topic: documentation", 
+                    "topic: packages"]
 
     f.write("Release Notes \n\n\n\n")
     f.write("Category " + "release notes: highlight" + "\n")
@@ -148,10 +163,16 @@ def main(startdate):
     repo = g.get_repo( orgName + "/" + repoName)
     
     print("Current GitHub API capacity", g.rate_limiting)
+    # TODO: also print timestamp
     prs = get_prs(repo,startdate)
+    # TODO: implement caching prs in a local file, and an option to use the cache or 
+    # to enforce retrieving updated PR details from Github. I think default is to update 
+    # from GitHub (to get newly merged PRs, updates of labels, PR titles etc., while the
+    # cache could be used for testing and polishing the code to generate output )
+    # 
     changes_overview(prs,startdate)
     print("Remaining GitHub API capacity", g.rate_limiting)
-
+    # TODO: also print timestamp
     
 if __name__ == "__main__":
     print("script name is", sys.argv[0])

From ee1dc764c94ecf44a6dc724c9d601b3fb1582922 Mon Sep 17 00:00:00 2001
From: Alexander Konovalov <alexk@mcs.st-andrews.ac.uk>
Date: Fri, 19 Feb 2021 21:11:50 +0000
Subject: [PATCH 04/23] Cache data locally to reduce calling GitHub API

---
 dev/release-notes.py | 30 +++++++++++++++++++++++-------
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/dev/release-notes.py b/dev/release-notes.py
index 1b5dd58027..e4e59533d9 100755
--- a/dev/release-notes.py
+++ b/dev/release-notes.py
@@ -4,11 +4,15 @@
 #
 
 import sys
+import json
+import os.path
 from github import Github
 from datetime import datetime
 
+
 def get_prs(repo,startdate):
-    """Returns a dictionary of PRs matching selection criteria."""
+    """Retrieves data for PRs matching selection criteria and puts them in a dictionary,
+       which is then saved in a json file, and also returned for immediate use."""
     prs = {}
     all_pulls = repo.get_pulls(state='closed', sort='created', direction='desc', base='master')
     # We need to run this over the whole list of PRs. Sorting by creation date descending
@@ -23,10 +27,12 @@ def get_prs(repo,startdate):
                 prs[pr.number] = { "title" : pr.title,
                                     "closed_at" : pr.closed_at.isoformat(),
                                     "labels" : labs }
-                if len(prs)>50: # for quick testing
-                    break
+#                 if len(prs)>5: # for quick testing (maybe later have an optional argument)
+#                     break
     print("\n")
-    return prs
+    with open('prscache.json', 'w', encoding='utf-8') as f:
+        json.dump(prs, f, ensure_ascii=False, indent=4)
+    return prs    
 
 def changes_overview(prs,startdate):
     """Writes files with information for release notes."""
@@ -58,6 +64,7 @@ def changes_overview(prs,startdate):
     for k in prs:
         # The format of an entry of list is: ['title of PR', 'Link' (Alternative the PR number can be used), [ list of labels ] ]
         if "release notes: highlight" in prs[k]["labels"]:
+            # TODO: writing up these details should be a function
             f.write("- [#")
             issuenumber = str(k)
             f.write(issuenumber)
@@ -164,12 +171,21 @@ def main(startdate):
     
     print("Current GitHub API capacity", g.rate_limiting)
     # TODO: also print timestamp
-    prs = get_prs(repo,startdate)
-    # TODO: implement caching prs in a local file, and an option to use the cache or 
+
+    # TODO: we cache PRs data in a local file. For now, if it exists, it will be used, 
+    # otherwise it will be recreated. Later, there may be an option to use the cache or 
     # to enforce retrieving updated PR details from Github. I think default is to update 
     # from GitHub (to get newly merged PRs, updates of labels, PR titles etc., while the
     # cache could be used for testing and polishing the code to generate output )
-    # 
+
+    if os.path.isfile("prscache.json"):
+        print("Using cached data from prscache.json ...")
+        with open("prscache.json", "r") as read_file:
+            prs = json.load(read_file)
+    else:    
+        print("Retriving data using GitHub API ...")
+        prs = get_prs(repo,startdate)
+  
     changes_overview(prs,startdate)
     print("Remaining GitHub API capacity", g.rate_limiting)
     # TODO: also print timestamp

From 7c8db17bf9d95da4c2dea8140b21e9772a4630b2 Mon Sep 17 00:00:00 2001
From: Alexander Konovalov <alexk@mcs.st-andrews.ac.uk>
Date: Fri, 19 Feb 2021 22:54:56 +0000
Subject: [PATCH 05/23] Consistently use double quotes

---
 dev/release-notes.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/dev/release-notes.py b/dev/release-notes.py
index e4e59533d9..660bc379ad 100755
--- a/dev/release-notes.py
+++ b/dev/release-notes.py
@@ -14,11 +14,11 @@ def get_prs(repo,startdate):
     """Retrieves data for PRs matching selection criteria and puts them in a dictionary,
        which is then saved in a json file, and also returned for immediate use."""
     prs = {}
-    all_pulls = repo.get_pulls(state='closed', sort='created', direction='desc', base='master')
+    all_pulls = repo.get_pulls(state="closed", sort="created", direction="desc", base="master")
     # We need to run this over the whole list of PRs. Sorting by creation date descending
     # is not really helping - could be that some very old PRs are being merged.
     for pr in all_pulls:
-        print(pr.number, end=' ')
+        print(pr.number, end=" ")
         # flush stdout immediately, to see progress indicator
         sys.stdout.flush()
         if pr.merged:
@@ -30,7 +30,7 @@ def get_prs(repo,startdate):
 #                 if len(prs)>5: # for quick testing (maybe later have an optional argument)
 #                     break
     print("\n")
-    with open('prscache.json', 'w', encoding='utf-8') as f:
+    with open("prscache.json", "w", encoding="utf-8") as f:
         json.dump(prs, f, ensure_ascii=False, indent=4)
     return prs    
 
@@ -62,7 +62,7 @@ def changes_overview(prs,startdate):
     f.write("Category " + "release notes: highlight" + "\n")
     removelist = []
     for k in prs:
-        # The format of an entry of list is: ['title of PR', 'Link' (Alternative the PR number can be used), [ list of labels ] ]
+        # The format of an entry of list is: ["title of PR", "Link" (Alternative the PR number can be used), [ list of labels ] ]
         if "release notes: highlight" in prs[k]["labels"]:
             # TODO: writing up these details should be a function
             f.write("- [#")
@@ -161,8 +161,8 @@ def main(startdate):
 
     # Authentication and checking current API capacity
     # TODO: for now this will do, use Sergio's code later
-    with open('/Users/alexk/.github_shell_token', 'r') as f:
-        accessToken=f.read().replace('\n', '')
+    with open("/Users/alexk/.github_shell_token", "r") as f:
+        accessToken=f.read().replace("\n", "")
     g=Github(accessToken)
 
     orgName = "gap-system"

From f1579efae8209f4865233d485702cedb737eb002 Mon Sep 17 00:00:00 2001
From: Alexander Konovalov <alexk@mcs.st-andrews.ac.uk>
Date: Fri, 19 Feb 2021 23:08:08 +0000
Subject: [PATCH 06/23] Add timestamp when printing API capacity

---
 dev/release-notes.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/dev/release-notes.py b/dev/release-notes.py
index 660bc379ad..741250cca3 100755
--- a/dev/release-notes.py
+++ b/dev/release-notes.py
@@ -169,8 +169,7 @@ def main(startdate):
     repoName = "gap"
     repo = g.get_repo( orgName + "/" + repoName)
     
-    print("Current GitHub API capacity", g.rate_limiting)
-    # TODO: also print timestamp
+    print("Current GitHub API capacity", g.rate_limiting, "at", datetime.now().isoformat() )
 
     # TODO: we cache PRs data in a local file. For now, if it exists, it will be used, 
     # otherwise it will be recreated. Later, there may be an option to use the cache or 
@@ -187,8 +186,7 @@ def main(startdate):
         prs = get_prs(repo,startdate)
   
     changes_overview(prs,startdate)
-    print("Remaining GitHub API capacity", g.rate_limiting)
-    # TODO: also print timestamp
+    print("Remaining GitHub API capacity", g.rate_limiting, "at", datetime.now().isoformat() )
     
 if __name__ == "__main__":
     print("script name is", sys.argv[0])

From e30d6fc5d6cd0e8b34f016d1a73350efe09fb92f Mon Sep 17 00:00:00 2001
From: Alexander Konovalov <alexk@mcs.st-andrews.ac.uk>
Date: Fri, 19 Feb 2021 23:16:59 +0000
Subject: [PATCH 07/23] Check the date format and provide usage instructions

---
 dev/release-notes.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/dev/release-notes.py b/dev/release-notes.py
index 741250cca3..27e89042a8 100755
--- a/dev/release-notes.py
+++ b/dev/release-notes.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Usage ./release-notes.py YYYY-MM-DD
+# Usage: ./release-notes.py YYYY-MM-DD
 #
 
 import sys
@@ -9,6 +9,8 @@
 from github import Github
 from datetime import datetime
 
+def usage():
+    print("Usage: ./release-notes.py YYYY-MM-DD")
 
 def get_prs(repo,startdate):
     """Retrieves data for PRs matching selection criteria and puts them in a dictionary,
@@ -189,8 +191,16 @@ def main(startdate):
     print("Remaining GitHub API capacity", g.rate_limiting, "at", datetime.now().isoformat() )
     
 if __name__ == "__main__":
-    print("script name is", sys.argv[0])
     if (len(sys.argv) != 2): # the argument is the start date in ISO 8601
-        usage()              # to be defined
-       	sys.exit(0)          # exit with return value of 0
+        usage()
+       	sys.exit(0)
+
+    try:
+        datetime.fromisoformat(sys.argv[1])
+        
+    except:
+        print("The date is not in ISO8601 format!")
+        usage()
+       	sys.exit(0)
+
     main(sys.argv[1])

From e0dde6715a76c633dab0f2cb056355587a557da5 Mon Sep 17 00:00:00 2001
From: Alexander Konovalov <alexk@mcs.st-andrews.ac.uk>
Date: Fri, 19 Feb 2021 23:36:50 +0000
Subject: [PATCH 08/23] Use more appropriate headers instead of labels

---
 dev/release-notes.py | 45 ++++++++++++++++++++++++--------------------
 1 file changed, 25 insertions(+), 20 deletions(-)

diff --git a/dev/release-notes.py b/dev/release-notes.py
index 27e89042a8..f2585faeb4 100755
--- a/dev/release-notes.py
+++ b/dev/release-notes.py
@@ -49,19 +49,24 @@ def changes_overview(prs,startdate):
     f3 = open("releasenotes.json", "w")
     jsondict = prs.copy()
 
-    prioritylist = ["kind: bug: wrong result", 
-                    "kind: bug: crash", 
-                    "kind: bug: unexpected error", 
-                    "kind: bug", "kind: enhancement", 
-                    "kind: new feature", 
-                    "kind: performance", 
-                    "topic:libgap", 
-                    "topic: julia", 
-                    "topic: documentation", 
-                    "topic: packages"]
-
-    f.write("Release Notes \n\n\n\n")
-    f.write("Category " + "release notes: highlight" + "\n")
+    prioritylist = [
+        ["kind: bug: wrong result", "Fixed bugs that could lead to incorrect results"],
+        ["kind: bug: crash", "Fixed bugs that could lead to crashes"],
+        ["kind: bug: unexpected error", "Fixed bugs that could lead to break loops"],
+        ["kind: bug", "Other fixed bugs"],
+        ["kind: enhancement", "Improved and extended functionality"],
+        ["kind: new feature", "New features"], 
+        ["kind: performance", "Performance improvements"],
+        ["topic:libgap", "Improvements in the experimental way to allow 3rd party code to link GAP as a library"],
+        ["topic: julia", "Improvements in the experimental support for using the **Julia** garbage collector"],
+        ["topic: documentation", "Changed documentation"],
+        ["topic: packages", "Packages"]
+    ]
+
+    # TODO: why does this need a special treatment? 
+    # Adding it to the prioritylist could ensure that it goes first
+    f.write("## Release Notes \n\n")
+    f.write("### " + "New features and major changes" + "\n")
     removelist = []
     for k in prs:
         # The format of an entry of list is: ["title of PR", "Link" (Alternative the PR number can be used), [ list of labels ] ]
@@ -78,7 +83,7 @@ def changes_overview(prs,startdate):
             removelist.append(k)
     for item in removelist:
         del prs[item]
-    f.write("\n\n\n")
+    f.write("\n")
 
 
     removelist = []
@@ -90,7 +95,7 @@ def changes_overview(prs,startdate):
         del jsondict[item]
 
 
-    f2.write("Category " + "release notes: to be added" + "\n")
+    f2.write("### " + "release notes: to be added" + "\n")
     removelist = []
     for k in prs:
         if "release notes: to be added" in prs[k]["labels"]:
@@ -105,10 +110,10 @@ def changes_overview(prs,startdate):
             removelist.append(k)
     for item in removelist:
         del prs[item]
-    f2.write("\n\n\n")
+    f2.write("\n")
 
 
-    f2.write("Uncategorized PR" + "\n")
+    f2.write("### Uncategorized PR" + "\n")
     removelist = []
     for k in prs:
         #if not "release notes: use title" in item[2]:
@@ -128,10 +133,10 @@ def changes_overview(prs,startdate):
     
 
     for priorityobject in prioritylist:
-        f.write("Category " + priorityobject + "\n")
+        f.write("### " + priorityobject[1] + "\n")
         removelist = []
         for k in prs:
-            if priorityobject in prs[k]["labels"]:
+            if priorityobject[0] in prs[k]["labels"]:
                 f.write("- [#")
                 issuenumber = str(k)
                 f.write(issuenumber)
@@ -143,7 +148,7 @@ def changes_overview(prs,startdate):
                 removelist.append(k)
         for item in removelist:
             del prs[item]
-        f.write("\n\n\n")
+        f.write("\n")
     f.close()
 
     f3.write("[")

From ebf72967857185ef6eb22b6f5bc38e450f2ce26e Mon Sep 17 00:00:00 2001
From: Alexander Konovalov <alex-konovalov@users.noreply.github.com>
Date: Wed, 24 Feb 2021 08:16:53 +0000
Subject: [PATCH 09/23] Apply suggestions from code review by @fingolfin

Co-authored-by: Max Horn <max@quendi.de>
---
 dev/release-notes.py | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/dev/release-notes.py b/dev/release-notes.py
index f2585faeb4..e787d5f6cd 100755
--- a/dev/release-notes.py
+++ b/dev/release-notes.py
@@ -49,6 +49,11 @@ def changes_overview(prs,startdate):
     f3 = open("releasenotes.json", "w")
     jsondict = prs.copy()
 
+    # the following is a list of pairs [LABEL, DESCRIPTION]; the first entry is the name of a GitHub label
+    # (be careful to match them precisely), the second is a headline for a section the release notes; any PR with
+    # the given label is put into the corresponding section; each PR is put into only one section, the first one
+    # one from this list it fits in.
+    # See also <https://github.com/gap-system/gap/issues/4257>.
     prioritylist = [
         ["kind: bug: wrong result", "Fixed bugs that could lead to incorrect results"],
         ["kind: bug: crash", "Fixed bugs that could lead to crashes"],
@@ -57,8 +62,8 @@ def changes_overview(prs,startdate):
         ["kind: enhancement", "Improved and extended functionality"],
         ["kind: new feature", "New features"], 
         ["kind: performance", "Performance improvements"],
-        ["topic:libgap", "Improvements in the experimental way to allow 3rd party code to link GAP as a library"],
-        ["topic: julia", "Improvements in the experimental support for using the **Julia** garbage collector"],
+        ["topic: libgap", "Improvements to the interface which allows 3rd party code to link GAP as a library"],
+        ["topic: julia", "Improvements in the support for using the **Julia** garbage collector"],
         ["topic: documentation", "Changed documentation"],
         ["topic: packages", "Packages"]
     ]
@@ -72,14 +77,8 @@ def changes_overview(prs,startdate):
         # The format of an entry of list is: ["title of PR", "Link" (Alternative the PR number can be used), [ list of labels ] ]
         if "release notes: highlight" in prs[k]["labels"]:
             # TODO: writing up these details should be a function
-            f.write("- [#")
-            issuenumber = str(k)
-            f.write(issuenumber)
-            f.write("](")
-            f.write("https://github.com/gap-system/gap/pull/" + str(k))
-            f.write(") ")
-            f.write(prs[k]["title"])
-            f.write("\n")
+            title = prs[k]["title"]
+            f.write(f"- [#{k}](https://github.com/gap-system/gap/pull/{k}) {title}\n")
             removelist.append(k)
     for item in removelist:
         del prs[item]
@@ -196,7 +195,7 @@ def main(startdate):
     print("Remaining GitHub API capacity", g.rate_limiting, "at", datetime.now().isoformat() )
     
 if __name__ == "__main__":
-    if (len(sys.argv) != 2): # the argument is the start date in ISO 8601
+    if len(sys.argv) != 2: # the argument is the start date in ISO 8601
         usage()
        	sys.exit(0)
 

From 62b32868e0f3607b99c8009a586e7bedd434d479 Mon Sep 17 00:00:00 2001
From: danielrademacher <33546089+danielrademacher@users.noreply.github.com>
Date: Wed, 24 Feb 2021 13:05:23 +0100
Subject: [PATCH 10/23] Description of script and API capacity

---
 dev/release-notes.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/dev/release-notes.py b/dev/release-notes.py
index e787d5f6cd..2485183bcc 100755
--- a/dev/release-notes.py
+++ b/dev/release-notes.py
@@ -1,7 +1,15 @@
 #!/usr/bin/env python3
 #
 # Usage: ./release-notes.py YYYY-MM-DD
+# 
+# Input:
+# A start date in the format YYYY-MM-DD. If one is in the folder of "release-notes.py", it can then be called using: python release-notes.py YYYY-MM-DD
 #
+# Output and description: 
+# This script is used to automatically generate the release notes based on the associated labels of pull requests 
+# that have been merged with the master branch since "startdate". 
+# This script gets the title, the PR number and the labels and categorizes them based on the priority list and discussion from #4257.
+# In addition, a file is generated of PR that could not be categorized and a file for the browse function by Thomas Breuer (see #4257).
 
 import sys
 import json
@@ -175,6 +183,8 @@ def main(startdate):
     repoName = "gap"
     repo = g.get_repo( orgName + "/" + repoName)
     
+    # There is a GitHub API capacity of 5000 per hour i.e. that a maximum of 5000 requests can be made to GitHub per hour.
+    # Therefore, the following line indicates how many requests are currently still available
     print("Current GitHub API capacity", g.rate_limiting, "at", datetime.now().isoformat() )
 
     # TODO: we cache PRs data in a local file. For now, if it exists, it will be used, 

From 293263a52d21e074fded5f186db12993b99ef997 Mon Sep 17 00:00:00 2001
From: Alexander Konovalov <alexk@mcs.st-andrews.ac.uk>
Date: Sat, 13 Mar 2021 17:08:08 +0000
Subject: [PATCH 11/23] Move release notes script and README to dev/releases

---
 dev/{release-notes.py => releases/generate_release_notes.py} | 0
 dev/{ => releases}/release_notes.readme.md                   | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename dev/{release-notes.py => releases/generate_release_notes.py} (100%)
 rename dev/{ => releases}/release_notes.readme.md (100%)

diff --git a/dev/release-notes.py b/dev/releases/generate_release_notes.py
similarity index 100%
rename from dev/release-notes.py
rename to dev/releases/generate_release_notes.py
diff --git a/dev/release_notes.readme.md b/dev/releases/release_notes.readme.md
similarity index 100%
rename from dev/release_notes.readme.md
rename to dev/releases/release_notes.readme.md

From 02fda8a54cc91c4b4d4c73f37d46fe9617d92dcb Mon Sep 17 00:00:00 2001
From: Alexander Konovalov <alexk@mcs.st-andrews.ac.uk>
Date: Sat, 13 Mar 2021 17:24:09 +0000
Subject: [PATCH 12/23] Move exiting with a non-zero code into usage()

---
 dev/releases/generate_release_notes.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/releases/generate_release_notes.py b/dev/releases/generate_release_notes.py
index 2485183bcc..c7c1afc1ee 100755
--- a/dev/releases/generate_release_notes.py
+++ b/dev/releases/generate_release_notes.py
@@ -19,6 +19,8 @@
 
 def usage():
     print("Usage: ./release-notes.py YYYY-MM-DD")
+    sys.exit(1)
+
 
 def get_prs(repo,startdate):
     """Retrieves data for PRs matching selection criteria and puts them in a dictionary,
@@ -207,7 +209,6 @@ def main(startdate):
 if __name__ == "__main__":
     if len(sys.argv) != 2: # the argument is the start date in ISO 8601
         usage()
-       	sys.exit(0)
 
     try:
         datetime.fromisoformat(sys.argv[1])
@@ -215,6 +216,5 @@ def main(startdate):
     except:
         print("The date is not in ISO8601 format!")
         usage()
-       	sys.exit(0)
 
     main(sys.argv[1])

From 5b5c03366e107544fdbb14504400ec317d087e26 Mon Sep 17 00:00:00 2001
From: Alexander Konovalov <alexk@mcs.st-andrews.ac.uk>
Date: Sat, 13 Mar 2021 18:20:44 +0000
Subject: [PATCH 13/23] Extend description in the comments

---
 dev/releases/generate_release_notes.py | 28 +++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/dev/releases/generate_release_notes.py b/dev/releases/generate_release_notes.py
index c7c1afc1ee..6641618ac1 100755
--- a/dev/releases/generate_release_notes.py
+++ b/dev/releases/generate_release_notes.py
@@ -1,15 +1,29 @@
 #!/usr/bin/env python3
 #
-# Usage: ./release-notes.py YYYY-MM-DD
+# Usage: ./generate_release_notes.py YYYY-MM-DD
 # 
-# Input:
-# A start date in the format YYYY-MM-DD. If one is in the folder of "release-notes.py", it can then be called using: python release-notes.py YYYY-MM-DD
+# Input: a starting date in the ISO-8601 format.
 #
 # Output and description: 
-# This script is used to automatically generate the release notes based on the associated labels of pull requests 
-# that have been merged with the master branch since "startdate". 
-# This script gets the title, the PR number and the labels and categorizes them based on the priority list and discussion from #4257.
-# In addition, a file is generated of PR that could not be categorized and a file for the browse function by Thomas Breuer (see #4257).
+# This script is used to automatically generate the release notes based on the labels of
+# pull requests that have been merged into the master branch since the starting date.
+# For each such pull request (PR), this script extracts from GitHub its title, number and
+# labels, using the GitHub API via the PyGithub package (https://github.com/PyGithub/PyGithub).
+# For API requests using Basic Authentication or OAuth, you can make up to 5,000 requests
+# per hour (https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting).
+# As of March 2020 this script consumes about 3400 API calls and runs for about 25 minutes.
+# This is why, to reduce the number of API calls and minimise the need to retrieve the data,
+# PR details will be stored in the file `prscache.json`, which will then be used to
+# categorise PR following the priority list and discussion from #4257, and output three
+# files:
+# - "releasenotes.md"   : list of PR by categories for adding to release notes
+# - "remainingPR.md"    : list of PR that could not be categorised
+# - "releasenotes.json" : data for `BrowseReleaseNotes` function by Thomas Breuer (see #4257).
+#
+# If this script detects the file `prscache.json` it will use it, otherwise it will retrieve
+# new data from GitHub. Thus, if new PR were merged, or there were updates of titles and labels
+# of merged PRs, you need to delete `prscache.json` to enforce updating local data (TODO: make
+# this turned on/off via a command line option in the next version).
 
 import sys
 import json

From 84b2abc259d4f0274159dd155124c4f24840784b Mon Sep 17 00:00:00 2001
From: Alexander Konovalov <alexk@mcs.st-andrews.ac.uk>
Date: Fri, 19 Mar 2021 22:24:38 +0000
Subject: [PATCH 14/23] Use formatted string to print link to PR in markdown

---
 dev/releases/generate_release_notes.py | 31 +++++---------------------
 1 file changed, 6 insertions(+), 25 deletions(-)

diff --git a/dev/releases/generate_release_notes.py b/dev/releases/generate_release_notes.py
index 6641618ac1..4c6cfdf999 100755
--- a/dev/releases/generate_release_notes.py
+++ b/dev/releases/generate_release_notes.py
@@ -100,7 +100,6 @@ def changes_overview(prs,startdate):
     for k in prs:
         # The format of an entry of list is: ["title of PR", "Link" (Alternative the PR number can be used), [ list of labels ] ]
         if "release notes: highlight" in prs[k]["labels"]:
-            # TODO: writing up these details should be a function
             title = prs[k]["title"]
             f.write(f"- [#{k}](https://github.com/gap-system/gap/pull/{k}) {title}\n")
             removelist.append(k)
@@ -122,14 +121,8 @@ def changes_overview(prs,startdate):
     removelist = []
     for k in prs:
         if "release notes: to be added" in prs[k]["labels"]:
-            f2.write("- [#")
-            issuenumber = str(k)
-            f2.write(issuenumber)
-            f2.write("](")
-            f2.write("https://github.com/gap-system/gap/pull/" + str(k))
-            f2.write(") ")
-            f2.write(prs[k]["title"])
-            f2.write("\n")
+            title = prs[k]["title"]
+            f2.write(f"- [#{k}](https://github.com/gap-system/gap/pull/{k}) {title}\n")
             removelist.append(k)
     for item in removelist:
         del prs[item]
@@ -141,14 +134,8 @@ def changes_overview(prs,startdate):
     for k in prs:
         #if not "release notes: use title" in item[2]:
         if not "release notes: added" in prs[k]["labels"]:
-            f2.write("- [#")
-            issuenumber = str(k)
-            f2.write(issuenumber)
-            f2.write("](")
-            f2.write("https://github.com/gap-system/gap/pull/" + str(k))
-            f2.write(") ")
-            f2.write(prs[k]["title"])
-            f2.write("\n")
+            title = prs[k]["title"]
+            f2.write(f"- [#{k}](https://github.com/gap-system/gap/pull/{k}) {title}\n")
             removelist.append(k)
     for item in removelist:
         del prs[item]
@@ -160,14 +147,8 @@ def changes_overview(prs,startdate):
         removelist = []
         for k in prs:
             if priorityobject[0] in prs[k]["labels"]:
-                f.write("- [#")
-                issuenumber = str(k)
-                f.write(issuenumber)
-                f.write("](")
-                f.write("https://github.com/gap-system/gap/pull/" + str(k))
-                f.write(") ")
-                f.write(prs[k]["title"])
-                f.write("\n")
+                title = prs[k]["title"]
+                f.write(f"- [#{k}](https://github.com/gap-system/gap/pull/{k}) {title}\n")
                 removelist.append(k)
         for item in removelist:
             del prs[item]

From 44e77c83fef4bea6fdd7e2cb7df55a1c2ce503a2 Mon Sep 17 00:00:00 2001
From: Alexander Konovalov <alexk@mcs.st-andrews.ac.uk>
Date: Sat, 20 Mar 2021 21:03:19 +0000
Subject: [PATCH 15/23] Next version of generating release note script

- now accepts only two arguments: 'minor' or 'major'
- the dates and other parameters are set in configuration variables in the script
- a function to filter PRs for minor/major release: further queries  may be added there
- tested by comparing these notes with the CHANGES.md for GAP 4.11.1
- As a result of the testing, added the output block "Other changes"
---
 dev/releases/generate_release_notes.py | 127 +++++++++++++++++++------
 1 file changed, 99 insertions(+), 28 deletions(-)

diff --git a/dev/releases/generate_release_notes.py b/dev/releases/generate_release_notes.py
index 4c6cfdf999..75f8ce6451 100755
--- a/dev/releases/generate_release_notes.py
+++ b/dev/releases/generate_release_notes.py
@@ -1,17 +1,22 @@
 #!/usr/bin/env python3
 #
-# Usage: ./generate_release_notes.py YYYY-MM-DD
+# Usage:
+#     ./generate_release_notes.py minor
+# or
+#     ./generate_release_notes.py major
 # 
-# Input: a starting date in the ISO-8601 format.
+# to specify the type of the release.
 #
 # Output and description: 
 # This script is used to automatically generate the release notes based on the labels of
-# pull requests that have been merged into the master branch since the starting date.
+# pull requests that have been merged into the master branch since the starting date
+# specified in the `history_start_date` variable below.
+#
 # For each such pull request (PR), this script extracts from GitHub its title, number and
 # labels, using the GitHub API via the PyGithub package (https://github.com/PyGithub/PyGithub).
 # For API requests using Basic Authentication or OAuth, you can make up to 5,000 requests
 # per hour (https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting).
-# As of March 2020 this script consumes about 3400 API calls and runs for about 25 minutes.
+# As of March 2021 this script consumes about 3400 API calls and runs for about 25 minutes.
 # This is why, to reduce the number of API calls and minimise the need to retrieve the data,
 # PR details will be stored in the file `prscache.json`, which will then be used to
 # categorise PR following the priority list and discussion from #4257, and output three
@@ -24,6 +29,10 @@
 # new data from GitHub. Thus, if new PR were merged, or there were updates of titles and labels
 # of merged PRs, you need to delete `prscache.json` to enforce updating local data (TODO: make
 # this turned on/off via a command line option in the next version).
+#
+# To find out when a branch was created, use e.g.
+#     git show --summary `git merge-base stable-4.11 master`
+#
 
 import sys
 import json
@@ -31,8 +40,34 @@
 from github import Github
 from datetime import datetime
 
+#############################################################################
+#
+# Configuration parameters
+#
+# the earliest date we need to track for the next minor/major releases
+history_start_date = "2019-09-09"
+
+# the date of the last minor release (later, we may need to have more precise timestamp
+# - maybe extracted from the corresponding release tag)
+minor_branch_start_date = "2020-03-01" # next day after the minor release
+# question: what if it was merged into master before 4.11.1, but backported after?
+# Hopefully, before publishing 4.11.1 we have backported everything that had to be
+# backported, so this was not the case.
+
+# this version number needed to form labels like "backport-to-4.11-DONE"
+minor_branch_version = "4.11"
+
+# not yet - will make sense after branching the `stable-4.12` branch:
+# major_branch_start_date = "2019-09-09"
+# major_branch_version = "4.12"
+# note that we will have to collate together PRs which are not backported to stable-4.11
+# between `history_start_date` and `major_branch_start_date`, and PRs backported to
+# stable-4.12 after `major_branch_start_date`
+#
+#############################################################################
+
 def usage():
-    print("Usage: ./release-notes.py YYYY-MM-DD")
+    print("Usage: `./release-notes.py minor` or `./release-notes.py major`")
     sys.exit(1)
 
 
@@ -48,7 +83,9 @@ def get_prs(repo,startdate):
         # flush stdout immediately, to see progress indicator
         sys.stdout.flush()
         if pr.merged:
-            if pr.closed_at > datetime.fromisoformat(startdate):
+            if pr.closed_at > datetime.fromisoformat(history_start_date):
+                # getting labels will cost further API calls - if the startdate is
+                # too far in the past, that may exceed the API capacity
                 labs = [lab.name for lab in list(pr.get_labels())]
                 prs[pr.number] = { "title" : pr.title,
                                     "closed_at" : pr.closed_at.isoformat(),
@@ -60,17 +97,39 @@ def get_prs(repo,startdate):
         json.dump(prs, f, ensure_ascii=False, indent=4)
     return prs    
 
-def changes_overview(prs,startdate):
-    """Writes files with information for release notes."""
 
-    #TODO: using cached data, check that the starting date is the same
-    # Also save the date when cache was saved (warning if old?)
-    print("## Changes since", startdate)
+def filter_prs(prs,rel_type):
+    newprs = {}
+
+    if rel_type == "minor":
+
+        # for minor release, list PRs backported to the stable-4.X brach since the previous minor release     
+        for k,v in sorted(prs.items()):
+            if "backport-to-" + minor_branch_version + "-DONE" in v["labels"]:
+                if datetime.fromisoformat(v["closed_at"]) > datetime.fromisoformat(minor_branch_start_date):
+                    newprs[k] = v
+        return newprs
+
+    elif rel_type == "major":
+
+        # for major release, list PRs not backported to the stable-4.X brach
+        for k,v in sorted(prs.items()):
+            if not "backport-to-" + minor_branch_version + "-DONE" in v["labels"]:
+                newprs[k] = v
+        return newprs
+
+    else:
+
+        usage()
+
+
+def changes_overview(prs,startdate,rel_type):
+    """Writes files with information for release notes."""
 
     # Opening files with "w" resets them
-    f = open("releasenotes.md", "w")
-    f2 = open("remainingPR.md", "w")
-    f3 = open("releasenotes.json", "w")
+    f = open("releasenotes_" + rel_type + ".md", "w")
+    f2 = open("remainingPR_" + rel_type + ".md", "w")
+    f3 = open("releasenotes_" + rel_type + ".json", "w")
     jsondict = prs.copy()
 
     # the following is a list of pairs [LABEL, DESCRIPTION]; the first entry is the name of a GitHub label
@@ -95,7 +154,7 @@ def changes_overview(prs,startdate):
     # TODO: why does this need a special treatment? 
     # Adding it to the prioritylist could ensure that it goes first
     f.write("## Release Notes \n\n")
-    f.write("### " + "New features and major changes" + "\n")
+    f.write("### " + "New features and major changes" + "\n\n")
     removelist = []
     for k in prs:
         # The format of an entry of list is: ["title of PR", "Link" (Alternative the PR number can be used), [ list of labels ] ]
@@ -143,7 +202,7 @@ def changes_overview(prs,startdate):
     
 
     for priorityobject in prioritylist:
-        f.write("### " + priorityobject[1] + "\n")
+        f.write("### " + priorityobject[1] + "\n\n")
         removelist = []
         for k in prs:
             if priorityobject[0] in prs[k]["labels"]:
@@ -153,6 +212,12 @@ def changes_overview(prs,startdate):
         for item in removelist:
             del prs[item]
         f.write("\n")
+
+    f.write("### Other changes \n\n")
+    for k in prs:
+        title = prs[k]["title"]
+        f.write(f"- [#{k}](https://github.com/gap-system/gap/pull/{k}) {title}\n")
+    f.write("\n")
     f.close()
 
     f3.write("[")
@@ -168,7 +233,8 @@ def changes_overview(prs,startdate):
     f3.write("]")
     f3.close
 
-def main(startdate):
+
+def main(rel_type):
 
     # Authentication and checking current API capacity
     # TODO: for now this will do, use Sergio's code later
@@ -184,32 +250,37 @@ def main(startdate):
     # Therefore, the following line indicates how many requests are currently still available
     print("Current GitHub API capacity", g.rate_limiting, "at", datetime.now().isoformat() )
 
+    # If this limit is exceeded, an exception will be raised:
+    # github.GithubException.RateLimitExceededException: 403
+    # {"message": "API rate limit exceeded for user ID XXX.", "documentation_url":
+    # "https://docs.github.com/rest/overview/resources-in-the-rest-api#rate-limiting"}
+
+
     # TODO: we cache PRs data in a local file. For now, if it exists, it will be used, 
     # otherwise it will be recreated. Later, there may be an option to use the cache or 
     # to enforce retrieving updated PR details from Github. I think default is to update 
     # from GitHub (to get newly merged PRs, updates of labels, PR titles etc., while the
     # cache could be used for testing and polishing the code to generate output )
 
+    # TODO: add some data to the cache, e.g. when the cache is saved.
+    # Produce warning if old.
+
     if os.path.isfile("prscache.json"):
         print("Using cached data from prscache.json ...")
         with open("prscache.json", "r") as read_file:
             prs = json.load(read_file)
     else:    
-        print("Retriving data using GitHub API ...")
-        prs = get_prs(repo,startdate)
+        print("Retrieving data using GitHub API ...")
+        prs = get_prs(repo,history_start_date)
   
-    changes_overview(prs,startdate)
+    prs = filter_prs(prs,rel_type)
+    changes_overview(prs,history_start_date,rel_type)
     print("Remaining GitHub API capacity", g.rate_limiting, "at", datetime.now().isoformat() )
+
     
 if __name__ == "__main__":
-    if len(sys.argv) != 2: # the argument is the start date in ISO 8601
-        usage()
-
-    try:
-        datetime.fromisoformat(sys.argv[1])
-        
-    except:
-        print("The date is not in ISO8601 format!")
+    # the argument is "minor" or "major" to specify release kind
+    if len(sys.argv) != 2 or not sys.argv[1] in ["minor","major"]:
         usage()
 
     main(sys.argv[1])

From 67296e8be70ff43eec571f6908fd6b8b067de784 Mon Sep 17 00:00:00 2001
From: Alexander Konovalov <alexk@mcs.st-andrews.ac.uk>
Date: Tue, 23 Mar 2021 17:26:05 +0000
Subject: [PATCH 16/23] Add comment to explain the output

---
 dev/releases/generate_release_notes.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dev/releases/generate_release_notes.py b/dev/releases/generate_release_notes.py
index 75f8ce6451..610d39a273 100755
--- a/dev/releases/generate_release_notes.py
+++ b/dev/releases/generate_release_notes.py
@@ -14,6 +14,7 @@
 #
 # For each such pull request (PR), this script extracts from GitHub its title, number and
 # labels, using the GitHub API via the PyGithub package (https://github.com/PyGithub/PyGithub).
+# To help to track the progress, it will output the number of the currently processed PR.
 # For API requests using Basic Authentication or OAuth, you can make up to 5,000 requests
 # per hour (https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting).
 # As of March 2021 this script consumes about 3400 API calls and runs for about 25 minutes.

From 5ffbb16b01f926935676a022078348ea6b701f36 Mon Sep 17 00:00:00 2001
From: Alexander Konovalov <alexk@mcs.st-andrews.ac.uk>
Date: Tue, 23 Mar 2021 18:46:50 +0000
Subject: [PATCH 17/23] Introduce pr_to_md(k, title) to print PR entry in
 markdown

---
 dev/releases/generate_release_notes.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/dev/releases/generate_release_notes.py b/dev/releases/generate_release_notes.py
index 610d39a273..4bef87ad81 100755
--- a/dev/releases/generate_release_notes.py
+++ b/dev/releases/generate_release_notes.py
@@ -124,6 +124,11 @@ def filter_prs(prs,rel_type):
         usage()
 
 
+def pr_to_md(k, title):
+    """Returns markdown string for the PR entry"""
+    return f"- [#{k}](https://github.com/gap-system/gap/pull/{k}) {title}\n"
+
+
 def changes_overview(prs,startdate,rel_type):
     """Writes files with information for release notes."""
 
@@ -160,8 +165,7 @@ def changes_overview(prs,startdate,rel_type):
     for k in prs:
         # The format of an entry of list is: ["title of PR", "Link" (Alternative the PR number can be used), [ list of labels ] ]
         if "release notes: highlight" in prs[k]["labels"]:
-            title = prs[k]["title"]
-            f.write(f"- [#{k}](https://github.com/gap-system/gap/pull/{k}) {title}\n")
+            f.write(pr_to_md(k, prs[k]["title"]))
             removelist.append(k)
     for item in removelist:
         del prs[item]
@@ -181,8 +185,7 @@ def changes_overview(prs,startdate,rel_type):
     removelist = []
     for k in prs:
         if "release notes: to be added" in prs[k]["labels"]:
-            title = prs[k]["title"]
-            f2.write(f"- [#{k}](https://github.com/gap-system/gap/pull/{k}) {title}\n")
+            f2.write(pr_to_md(k, prs[k]["title"]))
             removelist.append(k)
     for item in removelist:
         del prs[item]
@@ -194,8 +197,7 @@ def changes_overview(prs,startdate,rel_type):
     for k in prs:
         #if not "release notes: use title" in item[2]:
         if not "release notes: added" in prs[k]["labels"]:
-            title = prs[k]["title"]
-            f2.write(f"- [#{k}](https://github.com/gap-system/gap/pull/{k}) {title}\n")
+            f2.write(pr_to_md(k, prs[k]["title"]))
             removelist.append(k)
     for item in removelist:
         del prs[item]
@@ -207,8 +209,7 @@ def changes_overview(prs,startdate,rel_type):
         removelist = []
         for k in prs:
             if priorityobject[0] in prs[k]["labels"]:
-                title = prs[k]["title"]
-                f.write(f"- [#{k}](https://github.com/gap-system/gap/pull/{k}) {title}\n")
+                f.write(pr_to_md(k, prs[k]["title"]))
                 removelist.append(k)
         for item in removelist:
             del prs[item]
@@ -216,8 +217,7 @@ def changes_overview(prs,startdate,rel_type):
 
     f.write("### Other changes \n\n")
     for k in prs:
-        title = prs[k]["title"]
-        f.write(f"- [#{k}](https://github.com/gap-system/gap/pull/{k}) {title}\n")
+        f.write(pr_to_md(k, prs[k]["title"]))
     f.write("\n")
     f.close()
 

From 814490511083c2e67768ec10889bf6a958436c0c Mon Sep 17 00:00:00 2001
From: Alexander Konovalov <alexk@mcs.st-andrews.ac.uk>
Date: Tue, 23 Mar 2021 18:52:14 +0000
Subject: [PATCH 18/23] Update and move description of `prs` where it is used
 first

---
 dev/releases/generate_release_notes.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/dev/releases/generate_release_notes.py b/dev/releases/generate_release_notes.py
index 4bef87ad81..94162d6116 100755
--- a/dev/releases/generate_release_notes.py
+++ b/dev/releases/generate_release_notes.py
@@ -75,6 +75,20 @@ def usage():
 def get_prs(repo,startdate):
     """Retrieves data for PRs matching selection criteria and puts them in a dictionary,
        which is then saved in a json file, and also returned for immediate use."""
+    # The output `prs` is a dictionary with keys being PR numbers, and values being
+    # dictionaries with keys "title", "closed_at" and "labels", for example:
+    #
+    # "3355": {
+    #     "title": "Allow packages to use ISO 8601 dates in their PackageInfo.g",
+    #     "closed_at": "2021-02-20T15:44:48",
+    #     "labels": [
+    #         "gapdays2019-spring",
+    #         "gapsingular2019",
+    #         "kind: enhancement",
+    #         "release notes: to be added"
+    #     ]
+    # },
+  
     prs = {}
     all_pulls = repo.get_pulls(state="closed", sort="created", direction="desc", base="master")
     # We need to run this over the whole list of PRs. Sorting by creation date descending
@@ -163,7 +177,6 @@ def changes_overview(prs,startdate,rel_type):
     f.write("### " + "New features and major changes" + "\n\n")
     removelist = []
     for k in prs:
-        # The format of an entry of list is: ["title of PR", "Link" (Alternative the PR number can be used), [ list of labels ] ]
         if "release notes: highlight" in prs[k]["labels"]:
             f.write(pr_to_md(k, prs[k]["title"]))
             removelist.append(k)

From 053f5927f62683e3be82cd00c39f553272b96401 Mon Sep 17 00:00:00 2001
From: Alexander Konovalov <alexk@mcs.st-andrews.ac.uk>
Date: Tue, 23 Mar 2021 19:36:49 +0000
Subject: [PATCH 19/23] More informative names for files and their variables

---
 dev/releases/generate_release_notes.py | 55 +++++++++++++-------------
 1 file changed, 28 insertions(+), 27 deletions(-)

diff --git a/dev/releases/generate_release_notes.py b/dev/releases/generate_release_notes.py
index 94162d6116..260cb58557 100755
--- a/dev/releases/generate_release_notes.py
+++ b/dev/releases/generate_release_notes.py
@@ -22,9 +22,10 @@
 # PR details will be stored in the file `prscache.json`, which will then be used to
 # categorise PR following the priority list and discussion from #4257, and output three
 # files:
-# - "releasenotes.md"   : list of PR by categories for adding to release notes
-# - "remainingPR.md"    : list of PR that could not be categorised
-# - "releasenotes.json" : data for `BrowseReleaseNotes` function by Thomas Breuer (see #4257).
+# - "releasenotes_*.md"   : list of PR by categories for adding to release notes
+# - "unsorted_PRs_*.md"    : list of PR that could not be categorised
+# - "releasenotes_*.json" : data for `BrowseReleaseNotes` function by Thomas Breuer (see #4257).
+# where "*" is "minor" or "major" depending on the type of the release.
 #
 # If this script detects the file `prscache.json` it will use it, otherwise it will retrieve
 # new data from GitHub. Thus, if new PR were merged, or there were updates of titles and labels
@@ -147,9 +148,9 @@ def changes_overview(prs,startdate,rel_type):
     """Writes files with information for release notes."""
 
     # Opening files with "w" resets them
-    f = open("releasenotes_" + rel_type + ".md", "w")
-    f2 = open("remainingPR_" + rel_type + ".md", "w")
-    f3 = open("releasenotes_" + rel_type + ".json", "w")
+    relnotes_file = open("releasenotes_" + rel_type + ".md", "w")
+    unsorted_file = open("unsorted_PRs_" + rel_type + ".md", "w")
+    relnotes_json = open("releasenotes_" + rel_type + ".json", "w")
     jsondict = prs.copy()
 
     # the following is a list of pairs [LABEL, DESCRIPTION]; the first entry is the name of a GitHub label
@@ -173,16 +174,16 @@ def changes_overview(prs,startdate,rel_type):
 
     # TODO: why does this need a special treatment? 
     # Adding it to the prioritylist could ensure that it goes first
-    f.write("## Release Notes \n\n")
-    f.write("### " + "New features and major changes" + "\n\n")
+    relnotes_file.write("## Release Notes \n\n")
+    relnotes_file.write("### " + "New features and major changes" + "\n\n")
     removelist = []
     for k in prs:
         if "release notes: highlight" in prs[k]["labels"]:
-            f.write(pr_to_md(k, prs[k]["title"]))
+            relnotes_file.write(pr_to_md(k, prs[k]["title"]))
             removelist.append(k)
     for item in removelist:
         del prs[item]
-    f.write("\n")
+    relnotes_file.write("\n")
 
 
     removelist = []
@@ -194,47 +195,47 @@ def changes_overview(prs,startdate,rel_type):
         del jsondict[item]
 
 
-    f2.write("### " + "release notes: to be added" + "\n")
+    unsorted_file.write("### " + "release notes: to be added" + "\n")
     removelist = []
     for k in prs:
         if "release notes: to be added" in prs[k]["labels"]:
-            f2.write(pr_to_md(k, prs[k]["title"]))
+            unsorted_file.write(pr_to_md(k, prs[k]["title"]))
             removelist.append(k)
     for item in removelist:
         del prs[item]
-    f2.write("\n")
+    unsorted_file.write("\n")
 
 
-    f2.write("### Uncategorized PR" + "\n")
+    unsorted_file.write("### Uncategorized PR" + "\n")
     removelist = []
     for k in prs:
         #if not "release notes: use title" in item[2]:
         if not "release notes: added" in prs[k]["labels"]:
-            f2.write(pr_to_md(k, prs[k]["title"]))
+            unsorted_file.write(pr_to_md(k, prs[k]["title"]))
             removelist.append(k)
     for item in removelist:
         del prs[item]
-    f2.close()
+    unsorted_file.close()
     
 
     for priorityobject in prioritylist:
-        f.write("### " + priorityobject[1] + "\n\n")
+        relnotes_file.write("### " + priorityobject[1] + "\n\n")
         removelist = []
         for k in prs:
             if priorityobject[0] in prs[k]["labels"]:
-                f.write(pr_to_md(k, prs[k]["title"]))
+                relnotes_file.write(pr_to_md(k, prs[k]["title"]))
                 removelist.append(k)
         for item in removelist:
             del prs[item]
-        f.write("\n")
+        relnotes_file.write("\n")
 
-    f.write("### Other changes \n\n")
+    relnotes_file.write("### Other changes \n\n")
     for k in prs:
-        f.write(pr_to_md(k, prs[k]["title"]))
-    f.write("\n")
-    f.close()
+        relnotes_file.write(pr_to_md(k, prs[k]["title"]))
+    relnotes_file.write("\n")
+    relnotes_file.close()
 
-    f3.write("[")
+    relnotes_json.write("[")
     jsonlist = []
     for k in jsondict:
         temp = []
@@ -243,9 +244,9 @@ def changes_overview(prs,startdate,rel_type):
         temp.append(jsondict[k]["labels"])
         jsonlist.append(temp)
     for item in jsonlist:
-        f3.write("%s\n" % item)
-    f3.write("]")
-    f3.close
+        relnotes_json.write("%s\n" % item)
+    relnotes_json.write("]")
+    relnotes_json.close
 
 
 def main(rel_type):

From a3404fd863ea87b4a98f7295132145eab4bbc097 Mon Sep 17 00:00:00 2001
From: Alexander Konovalov <alexk@mcs.st-andrews.ac.uk>
Date: Tue, 23 Mar 2021 20:42:59 +0000
Subject: [PATCH 20/23] Switch to utils.initialize_github()

---
 dev/releases/generate_release_notes.py | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/dev/releases/generate_release_notes.py b/dev/releases/generate_release_notes.py
index 260cb58557..2104d1c75c 100755
--- a/dev/releases/generate_release_notes.py
+++ b/dev/releases/generate_release_notes.py
@@ -41,6 +41,8 @@
 import os.path
 from github import Github
 from datetime import datetime
+import utils
+
 
 #############################################################################
 #
@@ -251,15 +253,9 @@ def changes_overview(prs,startdate,rel_type):
 
 def main(rel_type):
 
-    # Authentication and checking current API capacity
-    # TODO: for now this will do, use Sergio's code later
-    with open("/Users/alexk/.github_shell_token", "r") as f:
-        accessToken=f.read().replace("\n", "")
-    g=Github(accessToken)
-
-    orgName = "gap-system"
-    repoName = "gap"
-    repo = g.get_repo( orgName + "/" + repoName)
+    utils.initialize_github()
+    g = utils.GITHUB_INSTANCE
+    repo = utils.CURRENT_REPO
     
     # There is a GitHub API capacity of 5000 per hour i.e. that a maximum of 5000 requests can be made to GitHub per hour.
     # Therefore, the following line indicates how many requests are currently still available

From 9b42c1a3094fca57bff249713259a54edcabe6a8 Mon Sep 17 00:00:00 2001
From: Alexander Konovalov <alexk@mcs.st-andrews.ac.uk>
Date: Tue, 23 Mar 2021 20:51:42 +0000
Subject: [PATCH 21/23] Add check for "release notes: use title" label

We need to use both old "release notes: added" label and
the newly introduced in "release notes: use title" label
since both label may appear in GAP 4.12.0 changes overview.
---
 dev/releases/generate_release_notes.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/dev/releases/generate_release_notes.py b/dev/releases/generate_release_notes.py
index 2104d1c75c..d4499d7820 100755
--- a/dev/releases/generate_release_notes.py
+++ b/dev/releases/generate_release_notes.py
@@ -211,8 +211,10 @@ def changes_overview(prs,startdate,rel_type):
     unsorted_file.write("### Uncategorized PR" + "\n")
     removelist = []
     for k in prs:
-        #if not "release notes: use title" in item[2]:
-        if not "release notes: added" in prs[k]["labels"]:
+        # we need to use both old "release notes: added" label and
+        # the newly introduced in "release notes: use title" label
+        # since both label may appear in GAP 4.12.0 changes overview
+        if not ("release notes: added" in prs[k]["labels"] or "release notes: use title" in prs[k]["labels"]):
             unsorted_file.write(pr_to_md(k, prs[k]["title"]))
             removelist.append(k)
     for item in removelist:

From 525f0eb250e6e3a0d35101e08e02b338bea32a78 Mon Sep 17 00:00:00 2001
From: Alexander Konovalov <alexk@mcs.st-andrews.ac.uk>
Date: Tue, 23 Mar 2021 21:56:35 +0000
Subject: [PATCH 22/23] Add "release notes: highlight" to priority list

This makes unnecessary its special treatment early in the script.
Also improved comments and instructions for the unsorted PRs.
---
 dev/releases/generate_release_notes.py | 43 ++++++++++++++------------
 1 file changed, 23 insertions(+), 20 deletions(-)

diff --git a/dev/releases/generate_release_notes.py b/dev/releases/generate_release_notes.py
index d4499d7820..60ccdb52d8 100755
--- a/dev/releases/generate_release_notes.py
+++ b/dev/releases/generate_release_notes.py
@@ -121,7 +121,7 @@ def filter_prs(prs,rel_type):
 
     if rel_type == "minor":
 
-        # for minor release, list PRs backported to the stable-4.X brach since the previous minor release     
+        # For minor release, list PRs backported to the stable-4.X brach since the previous minor release.
         for k,v in sorted(prs.items()):
             if "backport-to-" + minor_branch_version + "-DONE" in v["labels"]:
                 if datetime.fromisoformat(v["closed_at"]) > datetime.fromisoformat(minor_branch_start_date):
@@ -130,7 +130,10 @@ def filter_prs(prs,rel_type):
 
     elif rel_type == "major":
 
-        # for major release, list PRs not backported to the stable-4.X brach
+        # For major release, list PRs not backported to the stable-4.X brach.
+        # After branching stable-4.12 this will have to be changed to stop checking
+        # for "backport-to-4.11-DONE" at the date of the branching, and check for
+        # "backport-to-4.12-DONE" after that date
         for k,v in sorted(prs.items()):
             if not "backport-to-" + minor_branch_version + "-DONE" in v["labels"]:
                 newprs[k] = v
@@ -161,6 +164,7 @@ def changes_overview(prs,startdate,rel_type):
     # one from this list it fits in.
     # See also <https://github.com/gap-system/gap/issues/4257>.
     prioritylist = [
+        ["release notes: highlight", "New features and major changes"],
         ["kind: bug: wrong result", "Fixed bugs that could lead to incorrect results"],
         ["kind: bug: crash", "Fixed bugs that could lead to crashes"],
         ["kind: bug: unexpected error", "Fixed bugs that could lead to break loops"],
@@ -174,20 +178,9 @@ def changes_overview(prs,startdate,rel_type):
         ["topic: packages", "Packages"]
     ]
 
-    # TODO: why does this need a special treatment? 
-    # Adding it to the prioritylist could ensure that it goes first
-    relnotes_file.write("## Release Notes \n\n")
-    relnotes_file.write("### " + "New features and major changes" + "\n\n")
-    removelist = []
-    for k in prs:
-        if "release notes: highlight" in prs[k]["labels"]:
-            relnotes_file.write(pr_to_md(k, prs[k]["title"]))
-            removelist.append(k)
-    for item in removelist:
-        del prs[item]
-    relnotes_file.write("\n")
-
+    # Could also introduce some consistency checks here for wrong combinations of labels
 
+    # Drop PRs not needed for release notes
     removelist = []
     for k in prs:
         if "release notes: not needed" in prs[k]["labels"]:
@@ -196,8 +189,10 @@ def changes_overview(prs,startdate,rel_type):
         del prs[item]
         del jsondict[item]
 
-
-    unsorted_file.write("### " + "release notes: to be added" + "\n")
+    # Report PRs that have to be updated before inclusion into release notes.
+    unsorted_file.write("### " + "release notes: to be added" + "\n\n")
+    unsorted_file.write("If there are any PRs listed below, check their title and labels.\n")
+    unsorted_file.write("When done, change their label to \"release notes: use title\".\n\n")
     removelist = []
     for k in prs:
         if "release notes: to be added" in prs[k]["labels"]:
@@ -207,8 +202,10 @@ def changes_overview(prs,startdate,rel_type):
         del prs[item]
     unsorted_file.write("\n")
 
-
-    unsorted_file.write("### Uncategorized PR" + "\n")
+    # Report PRs that have neither "to be added" nor "added" or "use title" label
+    unsorted_file.write("### Uncategorized PR" + "\n\n")
+    unsorted_file.write("If there are any PRs listed below, either apply the same steps\n")
+    unsorted_file.write("as above, or change their label to \"release notes: not needed\".\n\n")
     removelist = []
     for k in prs:
         # we need to use both old "release notes: added" label and
@@ -220,7 +217,10 @@ def changes_overview(prs,startdate,rel_type):
     for item in removelist:
         del prs[item]
     unsorted_file.close()
-    
+
+    # All remaining PRs are to be included in the release notes
+
+    relnotes_file.write("## Release Notes \n\n")
 
     for priorityobject in prioritylist:
         relnotes_file.write("### " + priorityobject[1] + "\n\n")
@@ -233,6 +233,9 @@ def changes_overview(prs,startdate,rel_type):
             del prs[item]
         relnotes_file.write("\n")
 
+    # The remaining PRs have no "kind" or "topic" label from the priority list
+    # (may have other "kind" or "topic" label outside the priority list).
+    # Check their list in the release notes, and adjust labels if appropriate.
     relnotes_file.write("### Other changes \n\n")
     for k in prs:
         relnotes_file.write(pr_to_md(k, prs[k]["title"]))

From a4dab98d5cb5b0e1bb486d97e558366ce3b5a0bc Mon Sep 17 00:00:00 2001
From: Alexander Konovalov <alexk@mcs.st-andrews.ac.uk>
Date: Tue, 23 Mar 2021 22:06:57 +0000
Subject: [PATCH 23/23] Set date for the 4.11.2 chnages

---
 dev/releases/generate_release_notes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev/releases/generate_release_notes.py b/dev/releases/generate_release_notes.py
index 60ccdb52d8..1f5531a0e1 100755
--- a/dev/releases/generate_release_notes.py
+++ b/dev/releases/generate_release_notes.py
@@ -53,7 +53,7 @@
 
 # the date of the last minor release (later, we may need to have more precise timestamp
 # - maybe extracted from the corresponding release tag)
-minor_branch_start_date = "2020-03-01" # next day after the minor release
+minor_branch_start_date = "2021-03-03" # next day after the minor release (starts at midnight)
 # question: what if it was merged into master before 4.11.1, but backported after?
 # Hopefully, before publishing 4.11.1 we have backported everything that had to be
 # backported, so this was not the case.