feat: use a submodule for ninja sources

.github/workflows/build.yml

@@ -64,7 +64,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: 0  # required for versioneer to find tags
+          fetch-depth: 0  # required for setuptools_scm to find tags
+          submodules: 'recursive'
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3.2.0
@@ -97,7 +98,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: 0  # required for versioneer to find tags
+          fetch-depth: 0  # required for setuptools_scm to find tags
+          submodules: 'recursive'
 
       - name: Build SDist
         run: pipx run build --sdist

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "ninja-upstream"]
+	path = ninja-upstream
+	url = https://github.com/Kitware/ninja.git

CMakeLists.txt

@@ -5,56 +5,28 @@ project(NinjaPythonDistributions)
 set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_MODULE_PATH})
 
 # Options
-option(BUILD_VERBOSE "Display additional information while building (e.g download progress, ...)" OFF)
-option(RUN_NINJA_TEST "Run Ninja test suite" ON)
-set(ARCHIVE_DOWNLOAD_DIR "${CMAKE_BINARY_DIR}" CACHE PATH "Directory where to download archives")
+option(RUN_NINJA_TEST "Run Ninja test suite" OFF)
 
-include(NinjaUrls)
-
-function(check_archive_var archive_var)
-  if(NOT DEFINED "${archive_var}_url")
-    message(FATAL_ERROR "Failed to determine which archive to download: '${archive_var}_url' variable is not defined")
-  endif()
-  if(NOT DEFINED "${archive_var}_sha256")
-    message(FATAL_ERROR "Could you make sure variable '${archive_var}_sha256' is defined ?")
-  endif()
-endfunction()
-
-set(src_archive "unix_source")
-if(WIN32)
-  set(src_archive "windows_source")
-endif()
-check_archive_var("${src_archive}")
-
-message(STATUS "*********************************************")
+message(STATUS "************************************")
 message(STATUS "Ninja Python Distribution")
 message(STATUS "")
-message(STATUS "  BUILD_VERBOSE             : ${BUILD_VERBOSE}")
-message(STATUS "  RUN_NINJA_TEST            : ${RUN_NINJA_TEST}")
-message(STATUS "")
-message(STATUS "  ARCHIVE_DOWNLOAD_DIR      : ${ARCHIVE_DOWNLOAD_DIR}")
-message(STATUS "")
-message(STATUS "    src_archive             : ${src_archive}")
-message(STATUS "    <src_archive>_url       : ${${src_archive}_url}")
-message(STATUS "    <src_archive>_sha256    : ${${src_archive}_sha256}")
-message(STATUS "*********************************************")
-
-set(ep_download_no_progress_args)
-if(NOT BUILD_VERBOSE)
-  set(ep_download_no_progress_args
-    DOWNLOAD_NO_PROGRESS 1
-    )
-endif()
+message(STATUS "  RUN_NINJA_TEST : ${RUN_NINJA_TEST}")
+message(STATUS "************************************")
 
 include(FetchContent)
 FetchContent_Declare(
   ninja
-  URL ${${src_archive}_url}
-  URL_HASH SHA256=${${src_archive}_sha256}
-  DOWNLOAD_DIR ${ARCHIVE_DOWNLOAD_DIR}
-  ${ep_download_no_progress_args}
+  SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ninja-upstream
 )
 FetchContent_MakeAvailable(ninja)
 
+if(RUN_NINJA_TEST)
+  add_custom_target(
+    run_ninja_test ALL "$<TARGET_FILE:ninja_test>"
+    DEPENDS ninja_test
+    WORKING_DIRECTORY "$<TARGET_FILE_DIR:ninja_test>"
+  )
+endif()
+
 install(TARGETS ninja COMPONENT python DESTINATION "${SKBUILD_SCRIPTS_DIR}")
 install(FILES "${ninja_SOURCE_DIR}/misc/ninja_syntax.py" COMPONENT python DESTINATION ninja)

noxfile.py

@@ -95,16 +95,13 @@ def bump(session: nox.Session) -> None:
     else:
         version = args.version
 
-    deps = nox.project.load_toml("scripts/update_ninja_version.py")["dependencies"]
-    session.install(*deps)
-
     extra = ["--quiet"] if args.commit else []
     session.run("python", "scripts/update_ninja_version.py", "--upstream-repository", args.upstream_repository, version, *extra)
 
     if args.commit:
         session.run("git", "switch", "-c", f"update-to-ninja-{version}", external=True)
         files = (
-            "NinjaUrls.cmake",
+            "ninja-upstream",
             "README.rst",
             "tests/test_ninja.py",
             "docs/update_ninja_version.rst",

pyproject.toml

@@ -100,6 +100,13 @@ musllinux-ppc64le-image = "musllinux_1_1"
 musllinux-s390x-image = "musllinux_1_1"
 musllinux-armv7l-image = "musllinux_1_2"
 
+[tool.cibuildwheel.config-settings]
+"cmake.define.RUN_NINJA_TEST" = "ON"
+
+[[tool.cibuildwheel.overrides]]
+select = ["*-win_arm64",]
+config-settings."cmake.define.RUN_NINJA_TEST" = "OFF"
+
 [[tool.cibuildwheel.overrides]]
 select = "*-macos*"
 inherit.environment = "append"

scripts/update_ninja_version.py

@@ -1,26 +1,19 @@
-# /// script
-# dependencies = ["requests"]
-# ///
-
 """
-Command line executable allowing to update NinjaUrls.cmake, documentation
+Command line executable allowing to update upstream sources, documentation
 and tests given a Ninja version.
 """
 from __future__ import annotations
 
 import argparse
 import contextlib
-import hashlib
 import os
 import re
-import tempfile
+import shutil
+import subprocess
 import textwrap
+from pathlib import Path
 
-from requests import request
-
-ROOT_DIR = os.path.join(os.path.dirname(__file__), "..")
-
-REQ_BUFFER_SIZE = 65536  # Chunk size when iterating a download body
+ROOT_DIR = Path(__file__).parent.parent.resolve(strict=True)
 
 
 @contextlib.contextmanager
@@ -31,133 +24,57 @@ def _log(txt, verbose=True):
     if verbose:
         print(f"{txt} - done")
 
-
-def _download_file(download_url, filename):
-    response = request(
-        method='GET',
-        url=download_url,
-        allow_redirects=False,
-        headers={'Accept': 'application/octet-stream'},
-        stream=True)
-    while response.status_code == 302:
-        response = request(
-            'GET', response.headers['Location'], allow_redirects=False,
-            stream=True
-        )
-    with open(filename, 'w+b') as f:
-        for chunk in response.iter_content(chunk_size=REQ_BUFFER_SIZE):
-            f.write(chunk)
-
-    return filename
-
-
-def _hash_sum(filepath, algorithm="sha256", block_size=2 ** 20):
-    hasher = hashlib.new(algorithm)
-    with open(filepath, mode="rb") as fd:
-        while True:
-            data = fd.read(block_size)
-            if not data:
-                break
-            hasher.update(data)
-
-    return hasher.hexdigest()
-
-
-def _download_and_compute_sha256(url, filename):
-    filepath = os.path.join(tempfile.gettempdir(), filename)
-    with _log(f"Downloading {url}"):
-        _download_file(url, filepath)
-        sha256 = _hash_sum(filepath, algorithm="sha256")
-    return url, sha256
-
-
-def get_ninja_archive_urls_and_sha256s(upstream_repository, version, verbose=False):
-    tag_name = f"v{version}"
-    files_base_url = f"https://github.com/{upstream_repository}/archive/{tag_name}"
-
-    with _log(f"Collecting URLs and SHA256s from '{files_base_url}'"):
-
-        # Get SHA256s and URLs
-        urls = {
-            "unix_source": _download_and_compute_sha256(files_base_url + ".tar.gz", tag_name + ".tar.gz"),
-            "win_source": _download_and_compute_sha256(files_base_url + ".zip", tag_name + ".zip"),
-        }
-
-        if verbose:
-            for identifier, (url, sha256) in urls.items():
-                print(f"[{identifier}]\n{url}\n{sha256}\n")
-
-        return urls
-
-
-def generate_cmake_variables(urls_and_sha256s):
-    template_inputs = {}
-
-    # Get SHA256s and URLs
-    for var_prefix, urls_and_sha256s_values in urls_and_sha256s.items():
-        template_inputs[f"{var_prefix}_url"] = urls_and_sha256s_values[0]
-        template_inputs[f"{var_prefix}_sha256"] = urls_and_sha256s_values[1]
-
-    return textwrap.dedent(
-        """
-        #-----------------------------------------------------------------------------
-        # Ninja sources
-        set(unix_source_url       "{unix_source_url}")
-        set(unix_source_sha256    "{unix_source_sha256}")
-
-        set(windows_source_url    "{win_source_url}")
-        set(windows_source_sha256 "{win_source_sha256}")
-        """
-    ).format(**template_inputs)
-
-
-def update_cmake_urls_script(upstream_repository, version):
-    content = generate_cmake_variables(get_ninja_archive_urls_and_sha256s(upstream_repository, version))
-    cmake_urls_filename = "NinjaUrls.cmake"
-    cmake_urls_filepath = os.path.join(ROOT_DIR, cmake_urls_filename)
-
-    msg = f"Updating '{cmake_urls_filename}' with Ninja version {version}"
-    with _log(msg), open(cmake_urls_filepath, "w") as cmake_file:
-        cmake_file.write(content)
-
-
-def _update_file(filepath, regex, replacement, verbose=True):
+@contextlib.contextmanager
+def chdir(path: Path):
+    origin = Path().absolute()
+    os.chdir(path)
+    try:
+        yield
+    finally:
+        os.chdir(origin)
+
+
+def update_submodule(upstream_repository, version):
+    with chdir(ROOT_DIR):
+        subprocess.run(["git", "submodule", "deinit", "-f", "ninja-upstream"], check=True)
+        subprocess.run(["git", "rm", "-f", "ninja-upstream"], check=True)
+        shutil.rmtree(ROOT_DIR / ".git/modules/ninja-upstream")
+        subprocess.run(["git", "submodule", "add", f"https://github.com/{upstream_repository}.git", "ninja-upstream"], check=True)
+        subprocess.run(["git", "submodule", "update", "--init", "--recursive", "ninja-upstream"], check=True)
+        with chdir(ROOT_DIR / "ninja-upstream"):
+            subprocess.run(["git", "fetch", "--tags"], check=True)
+            subprocess.run(["git", "checkout", f"v{version}"], check=True)
+
+
+def _update_file(filepath: Path, regex, replacement, verbose=True):
     msg = f"Updating {os.path.relpath(filepath, ROOT_DIR)}"
     with _log(msg, verbose=verbose):
         pattern = re.compile(regex)
-        with open(filepath) as doc_file:
+        with filepath.open() as doc_file:
             lines = doc_file.readlines()
             updated_content = []
             for line in lines:
                 updated_content.append(re.sub(pattern, replacement, line))
-        with open(filepath, "w") as doc_file:
+        with filepath.open("w") as doc_file:
             doc_file.writelines(updated_content)
 
 
 def update_docs(upstream_repository, version):
     pattern = re.compile(r"ninja \d+.\d+.\d+(\.[\w\-]+)*")
     replacement = f"ninja {version}"
-    _update_file(
-        os.path.join(ROOT_DIR, "README.rst"),
-        pattern, replacement)
+    _update_file(ROOT_DIR / "README.rst", pattern, replacement)
 
     pattern = re.compile(r"(?<=v)\d+.\d+.\d+(?:\.[\w\-]+)*(?=(?:\.zip|\.tar\.gz|\/))")
     replacement = version
-    _update_file(
-        os.path.join(ROOT_DIR, "docs/update_ninja_version.rst"),
-        pattern, replacement)
+    _update_file(ROOT_DIR / "docs/update_ninja_version.rst", pattern, replacement)
 
     pattern = re.compile(r"(?<!v)\d+.\d+.\d+(?:\.[\w\-]+)*")
     replacement = version
-    _update_file(
-        os.path.join(ROOT_DIR, "docs/update_ninja_version.rst"),
-        pattern, replacement, verbose=False)
+    _update_file(ROOT_DIR / "docs/update_ninja_version.rst", pattern, replacement, verbose=False)
 
     pattern = re.compile(r"github\.com\/[\w\-_]+\/[\w\-_]+(?=\/(?:release|archive))")
     replacement = "github.com/" + upstream_repository
-    _update_file(
-        os.path.join(ROOT_DIR, "docs/update_ninja_version.rst"),
-        pattern, replacement, verbose=False)
+    _update_file(ROOT_DIR / "docs/update_ninja_version.rst", pattern, replacement, verbose=False)
 
 
 def update_tests(version):
@@ -170,8 +87,7 @@ def update_tests(version):
 
     pattern = re.compile(r'expected_version = "\d+.\d+.\d+(\.[\w\-]+)*"')
     replacement = f'expected_version = "{version}"'
-    _update_file(os.path.join(
-        ROOT_DIR, "tests/test_ninja.py"), pattern, replacement)
+    _update_file(ROOT_DIR / "tests/test_ninja.py", pattern, replacement)
 
 
 def main():
@@ -189,34 +105,27 @@ def main():
         default="Kitware/ninja",
         help="Ninja upstream repository",
     )
-    parser.add_argument(
-        "--collect-only",
-        action="store_true",
-        help="If specified, only display the archive URLs and associated hashsums",
-    )
     parser.add_argument(
         "--quiet",
         action="store_true",
         help="Hide the output",
     )
     args = parser.parse_args()
-    if args.collect_only:
-        get_ninja_archive_urls_and_sha256s(args.upstream_repository, args.ninja_version, verbose=True)
-    else:
-        update_cmake_urls_script(args.upstream_repository, args.ninja_version)
-        update_docs(args.upstream_repository, args.ninja_version)
-        update_tests(args.ninja_version)
-
-        if not args.quiet:
-            msg = """\
-                Complete! Now run:
-
-                git switch -c update-to-ninja-{release}
-                git add -u NinjaUrls.cmake docs/index.rst README.rst tests/test_ninja.py docs/update_ninja_version.rst
-                git commit -m "Update to Ninja {release}"
-                gh pr create --fill --body "Created by update_ninja_version.py"
-                """
-            print(textwrap.dedent(msg.format(release=args.ninja_version)))
+
+    update_submodule(args.upstream_repository, args.ninja_version)
+    update_docs(args.upstream_repository, args.ninja_version)
+    update_tests(args.ninja_version)
+
+    if not args.quiet:
+        msg = """\
+            Complete! Now run:
+
+            git switch -c update-to-ninja-{release}
+            git add -u ninja-upstream docs/index.rst README.rst tests/test_ninja.py docs/update_ninja_version.rst
+            git commit -m "Update to Ninja {release}"
+            gh pr create --fill --body "Created by update_ninja_version.py"
+            """
+        print(textwrap.dedent(msg.format(release=args.ninja_version)))
 
 
 if __name__ == "__main__":