Fix bug in git fetch, add more masterfiles versions, improve code qua…

…lity

Signed-off-by: jakub-nt <175944085+jakub-nt@users.noreply.github.com>

cfbs/masterfiles/analyze.py

@@ -1,4 +1,3 @@
-# TODO merge this with ENT-12099 branch cfbs analyze.py
 import os
 
 from cfbs.utils import dict_sorted_by_key, file_sha256

cfbs/masterfiles/download_all_versions.py

@@ -3,10 +3,9 @@
 
 from cfbs.utils import FetchError, fetch_url, get_json, mkdir, user_error
 
-ENTERPRISE_URL = "https://cfengine.com/release-data/enterprise/releases.json"
-COMMUNITY_URL = "https://cfengine.com/release-data/community/releases.json"
+ENTERPRISE_RELEASES_URL = "https://cfengine.com/release-data/enterprise/releases.json"
 
-ENTERPRISE_DOWNLOAD_PATH = "enterprise"
+DOWNLOAD_PATH = "downloaded_masterfiles"
 
 
 def get_download_urls_enterprise():
@@ -15,7 +14,7 @@ def get_download_urls_enterprise():
 
     print("* gathering download URLs...")
 
-    data = get_json(ENTERPRISE_URL)
+    data = get_json(ENTERPRISE_RELEASES_URL)
 
     for release_data in data["releases"]:
         version = release_data["version"]
@@ -54,7 +53,7 @@ def get_download_urls_enterprise():
         if masterfiles_data is None:
             # happens for 3.9.2, 3.9.0, 3.8.2, 3.8.1, 3.8.0, 3.7.4--3.6.2
             # 3.9.2: see above
-            # 3.9.0 and below: no masterfiles listed, and analogous unlisted URLs seemingly do not exist
+            # 3.9.0 and below: no masterfiles listed, and unlisted analogous URLs seemingly do not exist
             continue
 
         download_urls[version] = masterfiles_data["URL"]
@@ -63,10 +62,10 @@ def get_download_urls_enterprise():
     return download_urls, reported_checksums
 
 
-def download_versions_from_urls(output_path, download_urls, reported_checksums):
+def download_versions_from_urls(download_path, download_urls, reported_checksums):
     downloaded_versions = []
 
-    mkdir(output_path)
+    mkdir(download_path)
 
     for version, url in download_urls.items():
         # ignore master and .x versions
@@ -76,7 +75,7 @@ def download_versions_from_urls(output_path, download_urls, reported_checksums):
         print("* downloading from", url)
         downloaded_versions.append(version)
 
-        version_path = os.path.join(output_path, version)
+        version_path = os.path.join(download_path, version)
         mkdir(version_path)
 
         # download a version, and verify the reported checksum matches
@@ -91,19 +90,28 @@ def download_versions_from_urls(output_path, download_urls, reported_checksums):
         tarball_dir_path = os.path.join(version_path, "tarball")
         shutil.unpack_archive(tarball_path, tarball_dir_path)
 
-    return output_path, downloaded_versions
+    return downloaded_versions
 
 
-# TODO
-# def download_all_versions_community():
-#     data = get_json(COMMUNITY_URL)
-
-
-def download_all_versions_enterprise():
+def download_all_versions():
     download_urls, reported_checksums = get_download_urls_enterprise()
 
-    output_path, downloaded_versions = download_versions_from_urls(
-        ENTERPRISE_DOWNLOAD_PATH, download_urls, reported_checksums
+    # add masterfiles versions which do not appear in Enterprise releases but appear in Community releases
+    # 3.12.0b1
+    version = "3.12.0b1"
+    download_url = "https://cfengine-package-repos.s3.amazonaws.com/community_binaries/Community-3.12.0b1/misc/cfengine-masterfiles-3.12.0b1.pkg.tar.gz"
+    digest = "ede305dae7be3edfac04fc5b7f63b46adb3a5b1612f4755e855ee8e6b8d344d7"
+    download_urls[version] = download_url
+    reported_checksums[version] = digest
+    # 3.10.0b1
+    version = "3.10.0b1"
+    download_url = "https://cfengine-package-repos.s3.amazonaws.com/tarballs/cfengine-masterfiles-3.10.0b1.pkg.tar.gz"
+    digest = "09291617254705d79dea2531b23dbd0754f09029e90ce0b43b275aa02c1223a3"
+    download_urls[version] = download_url
+    reported_checksums[version] = digest
+
+    downloaded_versions = download_versions_from_urls(
+        DOWNLOAD_PATH, download_urls, reported_checksums
     )
 
-    return output_path, downloaded_versions
+    return DOWNLOAD_PATH, downloaded_versions

cfbs/masterfiles/generate_release_information.py

@@ -1,8 +1,4 @@
-# TODO document `cfbs generate-release-information`
-# it generates the .json data files in the cwd
-import sys
-
-from cfbs.masterfiles.download_all_versions import download_all_versions_enterprise
+from cfbs.masterfiles.download_all_versions import download_all_versions
 from cfbs.masterfiles.generate_vcf_download import generate_vcf_download
 from cfbs.masterfiles.generate_vcf_git_checkout import generate_vcf_git_checkout
 
@@ -11,16 +7,14 @@
 
 
 def generate_release_information():
-    print("Downloading Enterprise masterfiles...")
+    print("Downloading masterfiles...")
 
-    output_path, downloaded_versions = download_all_versions_enterprise()
-    # TODO Community coverage:
-    # downloaded_versions, reported_checksums = download_all_versions_community()
+    download_path, downloaded_versions = download_all_versions()
 
     print("Download finished. Every reported checksum matches.")
     print("Generating release information...")
 
-    generate_vcf_download(output_path, downloaded_versions)
+    generate_vcf_download(download_path, downloaded_versions)
     generate_vcf_git_checkout(downloaded_versions)
 
     # TODO automatic analysis of the difference between downloadable MPF data and git MPF data

cfbs/masterfiles/generate_vcf_git_checkout.py

@@ -28,7 +28,7 @@ def check_required_commands(commands):
         check_required_command(c)
 
 
-def generate_vcf_git_checkout(interesting_tags=None):
+def generate_vcf_git_checkout(checkout_tags):
     required_commands = ["git", "make", "automake", "autoconf"]
     check_required_commands(required_commands)
 
@@ -42,26 +42,13 @@ def generate_vcf_git_checkout(interesting_tags=None):
     else:
         subprocess.run(
             ["git", "fetch", "--all"],
-            cwd=DIR_PATH,
+            cwd=MPF_PATH,
             check=True,
         )
 
-    result = subprocess.run(
-        ["git", "tag"], cwd=MPF_PATH, capture_output=True, check=True
-    )
-    tags = result.stdout.decode("UTF-8").splitlines()
-
-    # if not given, choose tags to checkout - by default, only consider version releases
-    if interesting_tags is None:
-        interesting_tags = []
-
-        for tag in tags:
-            if "-" not in tag:
-                interesting_tags.append(tag)
-
     versions_dict, checksums_dict, files_dict = initialize_vcf()
 
-    for tag in interesting_tags:
+    for tag in checkout_tags:
         print("Checkouting tag", tag)
 
         # checkout the version
@@ -90,7 +77,7 @@ def generate_vcf_git_checkout(interesting_tags=None):
 
         # clean the files to prevent spillage to other versions
         subprocess.run(
-            ["git", "clean", "-dffx"],
+            ["git", "clean", "-dfx"],
             cwd=MPF_PATH,
             check=True,
             stdout=subprocess.DEVNULL,