From 4c02fbbf3f4b7bfedd56d19efde4977f344fd58d Mon Sep 17 00:00:00 2001 From: jakub-nt <175944085+jakub-nt@users.noreply.github.com> Date: Thu, 14 Nov 2024 20:41:11 +0100 Subject: [PATCH] Add checksums for unlisted URLs, verify checksums earlier Signed-off-by: jakub-nt <175944085+jakub-nt@users.noreply.github.com> --- cfbs/masterfiles/check_tarball_checksums.py | 36 ------------------- cfbs/masterfiles/download_all_versions.py | 28 ++++++++++----- .../generate_release_information.py | 14 +++----- 3 files changed, 23 insertions(+), 55 deletions(-) delete mode 100644 cfbs/masterfiles/check_tarball_checksums.py diff --git a/cfbs/masterfiles/check_tarball_checksums.py b/cfbs/masterfiles/check_tarball_checksums.py deleted file mode 100644 index a4b647f..0000000 --- a/cfbs/masterfiles/check_tarball_checksums.py +++ /dev/null @@ -1,36 +0,0 @@ -import os - -from cfbs.utils import file_sha256, immediate_files - - -def check_tarball_checksums(dir_path, downloaded_versions, reported_checksums): - does_match = True - - print("Verifying checksums...") - - for version in downloaded_versions: - if version in ("3.10.0", "3.9.2"): - # 3.10.0 lists a .tar.gz, not a .pkg.tar.gz - # 3.9.2 lists no masterfiles - continue - - version_path = os.path.join(dir_path, version) - - versions_files = immediate_files(version_path) - # the tarball should be the only file in the version's directory - tarball_name = versions_files[0] - - tarball_path = os.path.join(version_path, tarball_name) - - tarball_checksum = file_sha256(tarball_path) - - reported_checksum = reported_checksums[version] - - if tarball_checksum != reported_checksum: - does_match = False - print("* checksum difference:") - print(version) - print(tarball_checksum) - print(reported_checksum) - - return does_match diff --git a/cfbs/masterfiles/download_all_versions.py b/cfbs/masterfiles/download_all_versions.py index f0c3c64..651637c 100644 --- a/cfbs/masterfiles/download_all_versions.py +++ b/cfbs/masterfiles/download_all_versions.py @@ -1,7 +1,7 @@ import os import shutil -from cfbs.utils import fetch_url, get_json, mkdir +from cfbs.utils import FetchError, fetch_url, get_json, mkdir, user_error ENTERPRISE_URL = "https://cfengine.com/release-data/enterprise/releases.json" COMMUNITY_URL = "https://cfengine.com/release-data/community/releases.json" @@ -13,21 +13,27 @@ def get_download_urls_enterprise(): download_urls = {} reported_checksums = {} + print("* gathering download URLs...") + data = get_json(ENTERPRISE_URL) for release_data in data["releases"]: version = release_data["version"] if version == "3.10.0": - # for 3.10.0, for some reason, the masterfiles download link points to the .tar.gz tarball, rather than the .pkg.tar.gz tarball - # download the .pkg.tar.gz from an unlisted analoguous URL instead + # for 3.10.0, for some reason, the "Masterfiles ready-to-install tarball" is a .tar.gz tarball, rather than a .pkg.tar.gz tarball + # download the .pkg.tar.gz tarball from an unlisted analoguous URL instead download_url = "https://cfengine-package-repos.s3.amazonaws.com/tarballs/cfengine-masterfiles-3.10.0.pkg.tar.gz" + digest = "7b5e237529e11ce4ae295922dad1a681f13b95f3a7d247d39d3f5088f1a1d7d3" download_urls[version] = download_url + reported_checksums[version] = digest continue if version == "3.9.2": # for 3.9.2, no masterfiles are listed, but an unlisted analoguous URL exists download_url = "https://cfengine-package-repos.s3.amazonaws.com/tarballs/cfengine-masterfiles-3.9.2.pkg.tar.gz" + digest = "ae1a758530d4a4aad5b6812b61fc37ad1b5900b755f88a1ab98da7fd05a9f5cc" download_urls[version] = download_url + reported_checksums[version] = digest continue release_url = release_data["URL"] @@ -57,7 +63,7 @@ def get_download_urls_enterprise(): return download_urls, reported_checksums -def download_versions_from_urls(output_path, download_urls): +def download_versions_from_urls(output_path, download_urls, reported_checksums): downloaded_versions = [] mkdir(output_path) @@ -67,15 +73,20 @@ def download_versions_from_urls(output_path, download_urls): if url.startswith("http://buildcache"): continue - print("Downloading from", url) + print("* downloading from", url) downloaded_versions.append(version) version_path = os.path.join(output_path, version) mkdir(version_path) + # download a version, and verify the reported checksum matches filename = url.split("/")[-1] tarball_path = os.path.join(version_path, filename) - fetch_url(url, tarball_path) + checksum = reported_checksums[version] + try: + fetch_url(url, tarball_path, checksum) + except FetchError as e: + user_error("For version " + version + ": " + str(e)) tarball_dir_path = os.path.join(version_path, "tarball") shutil.unpack_archive(tarball_path, tarball_dir_path) @@ -92,8 +103,7 @@ def download_all_versions_enterprise(): download_urls, reported_checksums = get_download_urls_enterprise() output_path, downloaded_versions = download_versions_from_urls( - ENTERPRISE_DOWNLOAD_PATH, download_urls + ENTERPRISE_DOWNLOAD_PATH, download_urls, reported_checksums ) - # for local verification of the reported (Enterprise) (.pkg.tar.gz) checksums - return output_path, downloaded_versions, reported_checksums + return output_path, downloaded_versions diff --git a/cfbs/masterfiles/generate_release_information.py b/cfbs/masterfiles/generate_release_information.py index c89016f..35df228 100644 --- a/cfbs/masterfiles/generate_release_information.py +++ b/cfbs/masterfiles/generate_release_information.py @@ -3,7 +3,6 @@ import sys from cfbs.masterfiles.download_all_versions import download_all_versions_enterprise -from cfbs.masterfiles.check_tarball_checksums import check_tarball_checksums from cfbs.masterfiles.generate_vcf_download import generate_vcf_download from cfbs.masterfiles.generate_vcf_git_checkout import generate_vcf_git_checkout @@ -13,18 +12,13 @@ def generate_release_information(): print("Downloading Enterprise masterfiles...") - output_path, downloaded_versions, reported_checksums = ( - download_all_versions_enterprise() - ) + + output_path, downloaded_versions = download_all_versions_enterprise() # TODO Community coverage: # downloaded_versions, reported_checksums = download_all_versions_community() - # Enterprise 3.9.2 is downloaded but there is no reported checksum, so both args are necessary - if check_tarball_checksums(output_path, downloaded_versions, reported_checksums): - print("Every checksum matches") - else: - print("Checksums differ!") - sys.exit(1) + print("Download finished. Every reported checksum matches.") + print("Generating release information...") generate_vcf_download(output_path, downloaded_versions) generate_vcf_git_checkout(downloaded_versions)