From 83b6c6b4d67dfd30e99b6ae85f71e00048a40dee Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 1 Dec 2024 20:44:28 +0100 Subject: [PATCH 1/2] feat: use a submodule for ninja sources --- .github/workflows/build.yml | 6 +- .gitmodules | 3 + CMakeLists.txt | 44 +------- NinjaUrls.cmake | 8 -- constraints.txt | 2 - ninja-upstream | 1 + noxfile.py | 5 +- scripts/update_ninja_version.py | 191 +++++++++----------------------- 8 files changed, 64 insertions(+), 196 deletions(-) create mode 100644 .gitmodules delete mode 100644 NinjaUrls.cmake delete mode 100644 constraints.txt create mode 160000 ninja-upstream diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f8c1bb7..7d35c40 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,7 +64,8 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 0 # required for versioneer to find tags + fetch-depth: 0 # required for setuptools_scm to find tags + submodules: 'recursive' - name: Set up QEMU uses: docker/setup-qemu-action@v3.3.0 @@ -97,7 +98,8 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 0 # required for versioneer to find tags + fetch-depth: 0 # required for setuptools_scm to find tags + submodules: 'recursive' - name: Build SDist run: pipx run build --sdist diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..fa0442c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "ninja-upstream"] + path = ninja-upstream + url = https://github.com/Kitware/ninja.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 5f79690..5141271 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,52 +7,18 @@ set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_MODULE_PATH}) # Options option(BUILD_VERBOSE "Display additional information while building (e.g download progress, ...)" OFF) option(RUN_NINJA_TEST "Run Ninja test suite" ON) -set(ARCHIVE_DOWNLOAD_DIR "${CMAKE_BINARY_DIR}" CACHE PATH "Directory where to download archives") -include(NinjaUrls) - -function(check_archive_var archive_var) - if(NOT DEFINED "${archive_var}_url") - message(FATAL_ERROR "Failed to determine which archive to download: '${archive_var}_url' variable is not defined") - endif() - if(NOT DEFINED "${archive_var}_sha256") - message(FATAL_ERROR "Could you make sure variable '${archive_var}_sha256' is defined ?") - endif() -endfunction() - -set(src_archive "unix_source") -if(WIN32) - set(src_archive "windows_source") -endif() -check_archive_var("${src_archive}") - -message(STATUS "*********************************************") +message(STATUS "************************************") message(STATUS "Ninja Python Distribution") message(STATUS "") -message(STATUS " BUILD_VERBOSE : ${BUILD_VERBOSE}") -message(STATUS " RUN_NINJA_TEST : ${RUN_NINJA_TEST}") -message(STATUS "") -message(STATUS " ARCHIVE_DOWNLOAD_DIR : ${ARCHIVE_DOWNLOAD_DIR}") -message(STATUS "") -message(STATUS " src_archive : ${src_archive}") -message(STATUS " _url : ${${src_archive}_url}") -message(STATUS " _sha256 : ${${src_archive}_sha256}") -message(STATUS "*********************************************") - -set(ep_download_no_progress_args) -if(NOT BUILD_VERBOSE) - set(ep_download_no_progress_args - DOWNLOAD_NO_PROGRESS 1 - ) -endif() +message(STATUS " BUILD_VERBOSE : ${BUILD_VERBOSE}") +message(STATUS " RUN_NINJA_TEST : ${RUN_NINJA_TEST}") +message(STATUS "************************************") include(FetchContent) FetchContent_Declare( ninja - URL ${${src_archive}_url} - URL_HASH SHA256=${${src_archive}_sha256} - DOWNLOAD_DIR ${ARCHIVE_DOWNLOAD_DIR} - ${ep_download_no_progress_args} + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ninja-upstream ) FetchContent_MakeAvailable(ninja) diff --git a/NinjaUrls.cmake b/NinjaUrls.cmake deleted file mode 100644 index 7423b7f..0000000 --- a/NinjaUrls.cmake +++ /dev/null @@ -1,8 +0,0 @@ - -#----------------------------------------------------------------------------- -# Ninja sources -set(unix_source_url "https://github.com/Kitware/ninja/archive/v1.11.1.g95dee.kitware.jobserver-1.tar.gz") -set(unix_source_sha256 "7ba84551f5b315b4270dc7c51adef5dff83a2154a3665a6c9744245c122dd0db") - -set(windows_source_url "https://github.com/Kitware/ninja/archive/v1.11.1.g95dee.kitware.jobserver-1.zip") -set(windows_source_sha256 "d605f36bce5cf50799ad3a56e7d388d7c0cf6731db40f46334e076c7e036bfaa") diff --git a/constraints.txt b/constraints.txt deleted file mode 100644 index f8c847d..0000000 --- a/constraints.txt +++ /dev/null @@ -1,2 +0,0 @@ -ruamel.yaml==0.16.13 -ruamel.yaml.clib==0.2.2 diff --git a/ninja-upstream b/ninja-upstream new file mode 160000 index 0000000..95dee2a --- /dev/null +++ b/ninja-upstream @@ -0,0 +1 @@ +Subproject commit 95dee2a91d96c409d54f9fa0b70ea9aa2bdf8e63 diff --git a/noxfile.py b/noxfile.py index 6392934..c54064f 100644 --- a/noxfile.py +++ b/noxfile.py @@ -95,16 +95,13 @@ def bump(session: nox.Session) -> None: else: version = args.version - deps = nox.project.load_toml("scripts/update_ninja_version.py")["dependencies"] - session.install(*deps) - extra = ["--quiet"] if args.commit else [] session.run("python", "scripts/update_ninja_version.py", "--upstream-repository", args.upstream_repository, version, *extra) if args.commit: session.run("git", "switch", "-c", f"update-to-ninja-{version}", external=True) files = ( - "NinjaUrls.cmake", + "ninja-upstream", "README.rst", "tests/test_ninja.py", "docs/update_ninja_version.rst", diff --git a/scripts/update_ninja_version.py b/scripts/update_ninja_version.py index 5904037..0e68944 100644 --- a/scripts/update_ninja_version.py +++ b/scripts/update_ninja_version.py @@ -1,26 +1,19 @@ -# /// script -# dependencies = ["requests"] -# /// - """ -Command line executable allowing to update NinjaUrls.cmake, documentation +Command line executable allowing to update upstream sources, documentation and tests given a Ninja version. """ from __future__ import annotations import argparse import contextlib -import hashlib import os import re -import tempfile +import shutil +import subprocess import textwrap +from pathlib import Path -from requests import request - -ROOT_DIR = os.path.join(os.path.dirname(__file__), "..") - -REQ_BUFFER_SIZE = 65536 # Chunk size when iterating a download body +ROOT_DIR = Path(__file__).parent.parent.resolve(strict=True) @contextlib.contextmanager @@ -31,133 +24,57 @@ def _log(txt, verbose=True): if verbose: print(f"{txt} - done") - -def _download_file(download_url, filename): - response = request( - method='GET', - url=download_url, - allow_redirects=False, - headers={'Accept': 'application/octet-stream'}, - stream=True) - while response.status_code == 302: - response = request( - 'GET', response.headers['Location'], allow_redirects=False, - stream=True - ) - with open(filename, 'w+b') as f: - for chunk in response.iter_content(chunk_size=REQ_BUFFER_SIZE): - f.write(chunk) - - return filename - - -def _hash_sum(filepath, algorithm="sha256", block_size=2 ** 20): - hasher = hashlib.new(algorithm) - with open(filepath, mode="rb") as fd: - while True: - data = fd.read(block_size) - if not data: - break - hasher.update(data) - - return hasher.hexdigest() - - -def _download_and_compute_sha256(url, filename): - filepath = os.path.join(tempfile.gettempdir(), filename) - with _log(f"Downloading {url}"): - _download_file(url, filepath) - sha256 = _hash_sum(filepath, algorithm="sha256") - return url, sha256 - - -def get_ninja_archive_urls_and_sha256s(upstream_repository, version, verbose=False): - tag_name = f"v{version}" - files_base_url = f"https://github.com/{upstream_repository}/archive/{tag_name}" - - with _log(f"Collecting URLs and SHA256s from '{files_base_url}'"): - - # Get SHA256s and URLs - urls = { - "unix_source": _download_and_compute_sha256(files_base_url + ".tar.gz", tag_name + ".tar.gz"), - "win_source": _download_and_compute_sha256(files_base_url + ".zip", tag_name + ".zip"), - } - - if verbose: - for identifier, (url, sha256) in urls.items(): - print(f"[{identifier}]\n{url}\n{sha256}\n") - - return urls - - -def generate_cmake_variables(urls_and_sha256s): - template_inputs = {} - - # Get SHA256s and URLs - for var_prefix, urls_and_sha256s_values in urls_and_sha256s.items(): - template_inputs[f"{var_prefix}_url"] = urls_and_sha256s_values[0] - template_inputs[f"{var_prefix}_sha256"] = urls_and_sha256s_values[1] - - return textwrap.dedent( - """ - #----------------------------------------------------------------------------- - # Ninja sources - set(unix_source_url "{unix_source_url}") - set(unix_source_sha256 "{unix_source_sha256}") - - set(windows_source_url "{win_source_url}") - set(windows_source_sha256 "{win_source_sha256}") - """ - ).format(**template_inputs) - - -def update_cmake_urls_script(upstream_repository, version): - content = generate_cmake_variables(get_ninja_archive_urls_and_sha256s(upstream_repository, version)) - cmake_urls_filename = "NinjaUrls.cmake" - cmake_urls_filepath = os.path.join(ROOT_DIR, cmake_urls_filename) - - msg = f"Updating '{cmake_urls_filename}' with Ninja version {version}" - with _log(msg), open(cmake_urls_filepath, "w") as cmake_file: - cmake_file.write(content) - - -def _update_file(filepath, regex, replacement, verbose=True): +@contextlib.contextmanager +def chdir(path: Path): + origin = Path().absolute() + os.chdir(path) + try: + yield + finally: + os.chdir(origin) + + +def update_submodule(upstream_repository, version): + with chdir(ROOT_DIR): + subprocess.run(["git", "submodule", "deinit", "-f", "ninja-upstream"], check=True) + subprocess.run(["git", "rm", "-f", "ninja-upstream"], check=True) + shutil.rmtree(ROOT_DIR / ".git/modules/ninja-upstream") + subprocess.run(["git", "submodule", "add", f"https://github.com/{upstream_repository}.git", "ninja-upstream"], check=True) + subprocess.run(["git", "submodule", "update", "--init", "--recursive", "ninja-upstream"], check=True) + with chdir(ROOT_DIR / "ninja-upstream"): + subprocess.run(["git", "fetch", "--tags"], check=True) + subprocess.run(["git", "checkout", f"v{version}"], check=True) + + +def _update_file(filepath: Path, regex, replacement, verbose=True): msg = f"Updating {os.path.relpath(filepath, ROOT_DIR)}" with _log(msg, verbose=verbose): pattern = re.compile(regex) - with open(filepath) as doc_file: + with filepath.open() as doc_file: lines = doc_file.readlines() updated_content = [] for line in lines: updated_content.append(re.sub(pattern, replacement, line)) - with open(filepath, "w") as doc_file: + with filepath.open("w") as doc_file: doc_file.writelines(updated_content) def update_docs(upstream_repository, version): pattern = re.compile(r"ninja \d+.\d+.\d+(\.[\w\-]+)*") replacement = f"ninja {version}" - _update_file( - os.path.join(ROOT_DIR, "README.rst"), - pattern, replacement) + _update_file(ROOT_DIR / "README.rst", pattern, replacement) pattern = re.compile(r"(?<=v)\d+.\d+.\d+(?:\.[\w\-]+)*(?=(?:\.zip|\.tar\.gz|\/))") replacement = version - _update_file( - os.path.join(ROOT_DIR, "docs/update_ninja_version.rst"), - pattern, replacement) + _update_file(ROOT_DIR / "docs/update_ninja_version.rst", pattern, replacement) pattern = re.compile(r"(? Date: Mon, 2 Dec 2024 00:05:54 +0100 Subject: [PATCH 2/2] fix: respect RUN_NINJA_TEST option --- CMakeLists.txt | 12 +++++++++--- pyproject.toml | 7 +++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5141271..94757f6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,13 +5,11 @@ project(NinjaPythonDistributions) set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_MODULE_PATH}) # Options -option(BUILD_VERBOSE "Display additional information while building (e.g download progress, ...)" OFF) -option(RUN_NINJA_TEST "Run Ninja test suite" ON) +option(RUN_NINJA_TEST "Run Ninja test suite" OFF) message(STATUS "************************************") message(STATUS "Ninja Python Distribution") message(STATUS "") -message(STATUS " BUILD_VERBOSE : ${BUILD_VERBOSE}") message(STATUS " RUN_NINJA_TEST : ${RUN_NINJA_TEST}") message(STATUS "************************************") @@ -22,5 +20,13 @@ FetchContent_Declare( ) FetchContent_MakeAvailable(ninja) +if(RUN_NINJA_TEST) + add_custom_target( + run_ninja_test ALL "$" + DEPENDS ninja_test + WORKING_DIRECTORY "$" + ) +endif() + install(TARGETS ninja COMPONENT python DESTINATION "${SKBUILD_SCRIPTS_DIR}") install(FILES "${ninja_SOURCE_DIR}/misc/ninja_syntax.py" COMPONENT python DESTINATION ninja) diff --git a/pyproject.toml b/pyproject.toml index 8880d90..51d3058 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -100,6 +100,13 @@ musllinux-ppc64le-image = "musllinux_1_1" musllinux-s390x-image = "musllinux_1_1" musllinux-armv7l-image = "musllinux_1_2" +[tool.cibuildwheel.config-settings] +"cmake.define.RUN_NINJA_TEST" = "ON" + +[[tool.cibuildwheel.overrides]] +select = ["*-win_arm64",] +config-settings."cmake.define.RUN_NINJA_TEST" = "OFF" + [[tool.cibuildwheel.overrides]] select = "*-macos*" inherit.environment = "append"