diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b85a767..7efbbea 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,9 +38,9 @@ jobs: arch: "ppc64le" - os: ubuntu-20.04 arch: "s390x" - - os: windows-2019 + - os: windows-2016 arch: "AMD64" - - os: windows-2019 + - os: windows-2016 arch: "x86" - os: macos-10.15 arch: "x86_64" diff --git a/CMakeLists.txt b/CMakeLists.txt index 0a0ef10..0e7a1e9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,17 +5,9 @@ project(NinjaPythonDistributions) set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_MODULE_PATH}) # Options -set(_build_from_source_default ON) -if((WIN32 AND CMAKE_SIZEOF_VOID_P EQUAL 8) OR APPLE) - set(_build_from_source_default OFF) -endif() -option(BUILD_FROM_SOURCE "Build Ninja from source" ${_build_from_source_default}) option(BUILD_VERBOSE "Display additional information while building (e.g download progress, ...)" OFF) set(ARCHIVE_DOWNLOAD_DIR "${CMAKE_BINARY_DIR}" CACHE PATH "Directory where to download archives") -# Dependencies -find_package(PythonInterp REQUIRED) - include(NinjaUrls) #----------------------------------------------------------------------------- @@ -36,28 +28,12 @@ if(WIN32) endif() check_archive_var("${src_archive}") -set(binary_archive "linux32_binary") -if(CMAKE_SIZEOF_VOID_P EQUAL 8) - set(binary_archive "linux64_binary") -endif() -if(APPLE) - set(binary_archive "macosx_binary") -endif() -if(WIN32) - set(binary_archive "win32_binary") - if(CMAKE_SIZEOF_VOID_P EQUAL 8) - set(binary_archive "win64_binary") - endif() -endif() -check_archive_var("${binary_archive}") - #----------------------------------------------------------------------------- # Summary #----------------------------------------------------------------------------- message(STATUS "*********************************************") message(STATUS "Ninja Python Distribution") message(STATUS "") -message(STATUS " BUILD_FROM_SOURCE : ${BUILD_FROM_SOURCE}") message(STATUS " BUILD_VERBOSE : ${BUILD_VERBOSE}") message(STATUS "") message(STATUS " ARCHIVE_DOWNLOAD_DIR : ${ARCHIVE_DOWNLOAD_DIR}") @@ -65,10 +41,6 @@ message(STATUS "") message(STATUS " src_archive : ${src_archive}") message(STATUS " _url : ${${src_archive}_url}") message(STATUS " _sha256 : ${${src_archive}_sha256}") -message(STATUS "") -message(STATUS " binary_archive : ${binary_archive}") -message(STATUS " _url : ${${binary_archive}_url}") -message(STATUS " _sha256 : ${${binary_archive}_sha256}") message(STATUS "*********************************************") #----------------------------------------------------------------------------- @@ -105,18 +77,21 @@ elseif(NOT EXISTS ${Ninja_SOURCE_DIR}) message(FATAL_ERROR "Ninja_SOURCE_DIR is set to a nonexistent directory") endif() -if(BUILD_FROM_SOURCE) +#----------------------------------------------------------------------------- +# Build from source +#----------------------------------------------------------------------------- +if(UNIX AND NOT APPLE) + # We're more likely to build from sources from PyPI on UNIX + # In order to relax the constraint on CMake version, + # use the python bootstrap script rather than the CMake build (requires 3.15+) - #----------------------------------------------------------------------------- - # Build from source - #----------------------------------------------------------------------------- - set(ninja_executable ${Ninja_SOURCE_DIR}/ninja${CMAKE_EXECUTABLE_SUFFIX}) + # Dependencies + find_package(PythonInterp REQUIRED) + set(ninja_executable ${Ninja_SOURCE_DIR}/ninja${CMAKE_EXECUTABLE_SUFFIX}) set(bootstrap_command ${PYTHON_EXECUTABLE} configure.py --bootstrap) # Explicitly defining _BSD_SOURCE is required to support building the wheel on Alpine. See issue #22 - if(UNIX AND NOT APPLE) - set(bootstrap_command ${CMAKE_COMMAND} -E env CXXFLAGS=-D_BSD_SOURCE ${bootstrap_command}) - endif() + set(bootstrap_command ${CMAKE_COMMAND} -E env CXXFLAGS=-D_BSD_SOURCE ${bootstrap_command}) add_custom_command( COMMAND ${bootstrap_command} OUTPUT ${ninja_executable} @@ -125,41 +100,31 @@ if(BUILD_FROM_SOURCE) add_custom_target(build_ninja ALL DEPENDS download_ninja_source ${ninja_executable} ) - - find_program(STRIP_EXECUTABLE strip) - if(STRIP_EXECUTABLE) - add_custom_target(strip_ninja_executable ALL - COMMAND ${STRIP_EXECUTABLE} ${ninja_executable} - WORKING_DIRECTORY ${Ninja_SOURCE_DIR} - COMMENT "Stripping ninja executable" - ) - add_dependencies(strip_ninja_executable build_ninja) - endif() - else() - - #----------------------------------------------------------------------------- - # Download pre-built archive - #----------------------------------------------------------------------------- - if(${binary_archive}_sha256 STREQUAL "NA") - message(FATAL_ERROR "Pre-built archives not available for '${binary_archive}'. Consider setting BUILD_FROM_SOURCE to ON.") - endif() - # Download selected binary archive - ExternalProject_add(download_ninja_binary - SOURCE_DIR ${CMAKE_BINARY_DIR}/bin - URL ${${binary_archive}_url} - URL_HASH SHA256=${${binary_archive}_sha256} - DOWNLOAD_DIR ${ARCHIVE_DOWNLOAD_DIR} - USES_TERMINAL_DOWNLOAD 1 - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - BUILD_IN_SOURCE 1 + set(Ninja_BINARY_DIR ${CMAKE_BINARY_DIR}/Ninja-build) + ExternalProject_add(build_ninja + SOURCE_DIR ${Ninja_SOURCE_DIR} + BINARY_DIR ${Ninja_BINARY_DIR} + DOWNLOAD_COMMAND "" + UPDATE_COMMAND "" + BUILD_ALWAYS 1 + USES_TERMINAL_CONFIGURE 1 + USES_TERMINAL_BUILD 1 INSTALL_COMMAND "" - ${ep_download_no_progress_args} + DEPENDS + download_ninja_source ) - message(STATUS "download_ninja_binary - URL: ${${binary_archive}_url}") - set(ninja_executable ${CMAKE_BINARY_DIR}/bin/ninja${CMAKE_EXECUTABLE_SUFFIX}) + set(ninja_executable ${Ninja_BINARY_DIR}/ninja${CMAKE_EXECUTABLE_SUFFIX}) +endif() +find_program(STRIP_EXECUTABLE strip) +if(STRIP_EXECUTABLE) + add_custom_target(strip_ninja_executable ALL + COMMAND ${STRIP_EXECUTABLE} ${ninja_executable} + WORKING_DIRECTORY ${Ninja_SOURCE_DIR} + COMMENT "Stripping ninja executable" + ) + add_dependencies(strip_ninja_executable build_ninja) endif() install(FILES ${Ninja_SOURCE_DIR}/misc/ninja_syntax.py DESTINATION src/ninja) diff --git a/NinjaUrls.cmake b/NinjaUrls.cmake index b4c997c..2974e6c 100644 --- a/NinjaUrls.cmake +++ b/NinjaUrls.cmake @@ -1,25 +1,8 @@ #----------------------------------------------------------------------------- # Ninja sources -set(unix_source_url "https://github.com/kitware/ninja/archive/v1.10.0.gfb670.kitware.jobserver-1.tar.gz") +set(unix_source_url "https://github.com/Kitware/ninja/archive/v1.10.0.gfb670.kitware.jobserver-1.tar.gz") set(unix_source_sha256 "d00033813993116a4e14f835df813daee9916b107333d88dbb798a22f8671b1f") -set(windows_source_url "https://github.com/kitware/ninja/archive/v1.10.0.gfb670.kitware.jobserver-1.zip") +set(windows_source_url "https://github.com/Kitware/ninja/archive/v1.10.0.gfb670.kitware.jobserver-1.zip") set(windows_source_sha256 "8399607087a165436e6bb35beed7c235180ba4a9af608f8c19bc5812ced06a81") - -#----------------------------------------------------------------------------- -# Ninja binaries -set(linux32_binary_url "NA") # Linux 32-bit binaries not available -set(linux32_binary_sha256 "NA") - -set(linux64_binary_url "https://github.com/Kitware/ninja/releases/download/v1.10.0.gfb670.kitware.jobserver-1/ninja-1.10.0.gfb670.kitware.jobserver-1_x86_64-linux-gnu.tar.gz") -set(linux64_binary_sha256 "5bb10ae9df47489b0e77732ed9fd95acccdd211d01fb6b26130fd408515b8995") - -set(macosx_binary_url "https://github.com/Kitware/ninja/releases/download/v1.10.0.gfb670.kitware.jobserver-1/ninja-1.10.0.gfb670.kitware.jobserver-1_x86_64-apple-darwin.tar.gz") -set(macosx_binary_sha256 "4acd065772228a5dd4cf85428b304cc157f07c333e4ea9931978eaef59f5e313") - -set(win32_binary_url "NA") # Windows 32-bit binaries not available -set(win32_binary_sha256 "NA") - -set(win64_binary_url "https://github.com/Kitware/ninja/releases/download/v1.10.0.gfb670.kitware.jobserver-1/ninja-1.10.0.gfb670.kitware.jobserver-1_i686-pc-windows-msvc.zip") -set(win64_binary_sha256 "7f54cd8d72306b14bcb3518e21bc0c5f9d8ec31dea7ccd340cbfade070e30019") diff --git a/docs/update_ninja_version.rst b/docs/update_ninja_version.rst index 9d28c9c..d78bfff 100644 --- a/docs/update_ninja_version.rst +++ b/docs/update_ninja_version.rst @@ -7,7 +7,7 @@ Updating the Ninja version A developer should use the following steps to update the version ``X.Y.Z`` of Ninja associated with the current Ninja python distributions. -Available Ninja archives can be found `here `_. +Available Ninja archives can be found `here `_. Nox prodedure ------------- @@ -32,18 +32,12 @@ Classic procedure: $ release=1.10.0.gfb670.kitware.jobserver-1 $ python scripts/update_ninja_version.py ${release} - Collecting URLs and SHA256s from 'https://github.com/kitware/ninja/releases' - Downloading https://github.com/kitware/ninja/archive/v1.10.0.gfb670.kitware.jobserver-1.tar.gz - Downloading https://github.com/kitware/ninja/archive/v1.10.0.gfb670.kitware.jobserver-1.tar.gz - done - Downloading https://github.com/kitware/ninja/archive/v1.10.0.gfb670.kitware.jobserver-1.zip - Downloading https://github.com/kitware/ninja/archive/v1.10.0.gfb670.kitware.jobserver-1.zip - done - Downloading https://github.com/kitware/ninja/releases/download/v1.10.0.gfb670.kitware.jobserver-1/ninja-1.10.0.gfb670.kitware.jobserver-1 - Downloading https://github.com/kitware/ninja/releases/download/v1.10.0.gfb670.kitware.jobserver-1/ninja-1.10.0.gfb670.kitware.jobserver-1 - done - Downloading https://github.com/kitware/ninja/releases/download/v1.10.0.gfb670.kitware.jobserver-1/ninja-1.10.0.gfb670.kitware.jobserver-1 - Downloading https://github.com/kitware/ninja/releases/download/v1.10.0.gfb670.kitware.jobserver-1/ninja-1.10.0.gfb670.kitware.jobserver-1 - done - Downloading https://github.com/kitware/ninja/releases/download/v1.10.0.gfb670.kitware.jobserver-1/ninja-1.10.0.gfb670.kitware.jobserver-1 - Downloading https://github.com/kitware/ninja/releases/download/v1.10.0.gfb670.kitware.jobserver-1/ninja-1.10.0.gfb670.kitware.jobserver-1 - done - Collecting URLs and SHA256s from 'https://github.com/kitware/ninja/releases' - done + Collecting URLs and SHA256s from 'https://github.com/Kitware/ninja/archive/v1.10.0.gfb670.kitware.jobserver-1' + Downloading https://github.com/Kitware/ninja/archive/v1.10.0.gfb670.kitware.jobserver-1.tar.gz + Downloading https://github.com/Kitware/ninja/archive/v1.10.0.gfb670.kitware.jobserver-1.tar.gz - done + Downloading https://github.com/Kitware/ninja/archive/v1.10.0.gfb670.kitware.jobserver-1.zip + Downloading https://github.com/Kitware/ninja/archive/v1.10.0.gfb670.kitware.jobserver-1.zip - done + Collecting URLs and SHA256s from 'https://github.com/Kitware/ninja/archive/v1.10.0.gfb670.kitware.jobserver-1' - done Updating 'NinjaUrls.cmake' with CMake version 1.10.0.gfb670.kitware.jobserver-1 Updating 'NinjaUrls.cmake' with CMake version 1.10.0.gfb670.kitware.jobserver-1 - done Updating README.rst diff --git a/noxfile.py b/noxfile.py index 287f6ca..a7c0587 100644 --- a/noxfile.py +++ b/noxfile.py @@ -61,6 +61,13 @@ def bump(session: nox.Session) -> None: Set to a new version, use -- , otherwise will use the latest version. """ parser = argparse.ArgumentParser(description="Process some integers.") + parser.add_argument( + "--upstream-repository", + metavar="UPSTREAM_REPOSITORY", + choices=["Kitware/ninja", "ninja-build/ninja"], + default="Kitware/ninja", + help="Ninja upstream repository", + ) parser.add_argument( "--commit", action="store_true", help="Make a branch and commit." ) @@ -72,17 +79,17 @@ def bump(session: nox.Session) -> None: if args.version is None: session.install("lastversion") version = session.run( - "lastversion", "--format", "tag", "kitware/ninja", log=False, silent=True + "lastversion", "--format", "tag", args.upstream_repository, log=False, silent=True ).strip() if version.startswith("v"): version = version[1:] else: version = args.version - session.install("githubrelease") + session.install("requests") extra = ["--quiet"] if args.commit else [] - session.run("python", "scripts/update_ninja_version.py", version, *extra) + session.run("python", "scripts/update_ninja_version.py", "--upstream-repository", args.upstream_repository, version, *extra) if args.commit: session.run("git", "switch", "-c", f"update-to-ninja-{version}", external=True) diff --git a/scripts/update_ninja_version.py b/scripts/update_ninja_version.py index bbd2f12..cfcafde 100644 --- a/scripts/update_ninja_version.py +++ b/scripts/update_ninja_version.py @@ -1,43 +1,30 @@ +#!/usr/bin/env python3 # -*- coding: utf-8 -*- -"""Command line executable allowing to update NinjaUrls.cmake, documentation +""" +Command line executable allowing to update NinjaUrls.cmake, documentation and tests given a Ninja version. """ import argparse import contextlib import hashlib -import io import os import re import tempfile import textwrap try: - import github_release as ghr + from requests import request except ImportError: raise SystemExit( - "github_release not available: " - "consider installing it running 'pip install -U githubrelease'" + "requests not available: " + "consider installing it running 'pip install requests'" ) -from requests import request - ROOT_DIR = os.path.join(os.path.dirname(__file__), "..") REQ_BUFFER_SIZE = 65536 # Chunk size when iterating a download body -NINJA_RELEASES_GITHUB_REPO = "kitware/ninja" - -NINJA_SRC_ARCHIVE_URL_TEMPLATE = \ - "https://github.com/" + NINJA_RELEASES_GITHUB_REPO + "/archive/%s" - - -class NinjaReleaseNotFound(Exception): - def __init__(self, release_name): - super(NinjaReleaseNotFound, self).__init__( - "GitHub repository '{}': Couldn't find release '{}'".format( - NINJA_RELEASES_GITHUB_REPO, release_name)) - @contextlib.contextmanager def _log(txt, verbose=True): @@ -69,7 +56,7 @@ def _download_file(download_url, filename): def _hash_sum(filepath, algorithm="sha256", block_size=2 ** 20): hasher = hashlib.new(algorithm) - with io.open(filepath, mode="rb") as fd: + with open(filepath, mode="rb") as fd: while True: data = fd.read(block_size) if not data: @@ -87,53 +74,21 @@ def _download_and_compute_sha256(url, filename): return url, sha256 -def get_ninja_archive_urls_and_sha256s(version): - files_base_url = \ - "https://github.com/%s/releases" % NINJA_RELEASES_GITHUB_REPO +def get_ninja_archive_urls_and_sha256s(upstream_repository, version, verbose=False): + tag_name = f"v{version}" + files_base_url = f"https://github.com/{upstream_repository}/archive/{tag_name}" with _log("Collecting URLs and SHA256s from '%s'" % files_base_url): - tag_name = "v%s" % version - release = ghr.get_release(NINJA_RELEASES_GITHUB_REPO, tag_name) - if release is None: - raise NinjaReleaseNotFound(tag_name) - # Get SHA256s and URLs urls = { - "unix_source": _download_and_compute_sha256( - NINJA_SRC_ARCHIVE_URL_TEMPLATE % (tag_name + ".tar.gz"), - tag_name + ".tar.gz"), - "win_source": _download_and_compute_sha256( - NINJA_SRC_ARCHIVE_URL_TEMPLATE % (tag_name + ".zip"), - tag_name + ".zip") + "unix_source": _download_and_compute_sha256(files_base_url + ".tar.gz", tag_name + ".tar.gz"), + "win_source": _download_and_compute_sha256(files_base_url + ".zip", tag_name + ".zip"), } - if NINJA_RELEASES_GITHUB_REPO == "ninja-build/ninja": - expected = { - "ninja-linux.zip": "linux64_binary", - "ninja-mac.zip": "macosx_binary", - "ninja-win.zip": "win64_binary", - } - else: - expected = { - "ninja-%s_x86_64-linux-gnu.tar.gz" % version: "linux64_binary", - "ninja-%s_x86_64-apple-darwin.tar.gz" % version: "macosx_binary", - "ninja-%s_i686-pc-windows-msvc.zip" % version: "win64_binary", - } - - found = 0 - for asset in release["assets"]: - filename = asset["name"] - if filename not in expected: - continue - found += 1 - assert "browser_download_url" in asset - download_url = asset["browser_download_url"] - var_prefix = expected[filename] - urls[var_prefix] = \ - _download_and_compute_sha256(download_url, filename) - - assert len(expected) == found + if verbose: + for identifier, (url, sha256) in urls.items(): + print("[{}]\n{}\n{}\n".format(identifier, url, sha256)) return urls @@ -146,7 +101,8 @@ def generate_cmake_variables(urls_and_sha256s): template_inputs["%s_url" % var_prefix] = urls_and_sha256s[0] template_inputs["%s_sha256" % var_prefix] = urls_and_sha256s[1] - cmake_variables = textwrap.dedent(""" + cmake_variables = textwrap.dedent( + """ #----------------------------------------------------------------------------- # Ninja sources set(unix_source_url "{unix_source_url}") @@ -154,35 +110,18 @@ def generate_cmake_variables(urls_and_sha256s): set(windows_source_url "{win_source_url}") set(windows_source_sha256 "{win_source_sha256}") - - #----------------------------------------------------------------------------- - # Ninja binaries - set(linux32_binary_url "NA") # Linux 32-bit binaries not available - set(linux32_binary_sha256 "NA") - - set(linux64_binary_url "{linux64_binary_url}") - set(linux64_binary_sha256 "{linux64_binary_sha256}") - - set(macosx_binary_url "{macosx_binary_url}") - set(macosx_binary_sha256 "{macosx_binary_sha256}") - - set(win32_binary_url "NA") # Windows 32-bit binaries not available - set(win32_binary_sha256 "NA") - - set(win64_binary_url "{win64_binary_url}") - set(win64_binary_sha256 "{win64_binary_sha256}") - """).format(**template_inputs) + """ + ).format(**template_inputs) return cmake_variables -def update_cmake_urls_script(version): - content = generate_cmake_variables( - get_ninja_archive_urls_and_sha256s(version)) +def update_cmake_urls_script(upstream_repository, version): + content = generate_cmake_variables(get_ninja_archive_urls_and_sha256s(upstream_repository, version)) cmake_urls_filename = "NinjaUrls.cmake" cmake_urls_filepath = os.path.join(ROOT_DIR, cmake_urls_filename) - msg = "Updating '{}' with CMake version {}".format(cmake_urls_filename, version) + msg = "Updating '{}' with Ninja version {}".format(cmake_urls_filename, version) with _log(msg), open(cmake_urls_filepath, "w") as cmake_file: cmake_file.write(content) @@ -191,17 +130,16 @@ def _update_file(filepath, regex, replacement, verbose=True): msg = "Updating %s" % os.path.relpath(filepath, ROOT_DIR) with _log(msg, verbose=verbose): pattern = re.compile(regex) - with open(filepath, 'r') as doc_file: + with open(filepath, "r") as doc_file: lines = doc_file.readlines() updated_content = [] for line in lines: - updated_content.append( - re.sub(pattern, replacement, line)) + updated_content.append(re.sub(pattern, replacement, line)) with open(filepath, "w") as doc_file: doc_file.writelines(updated_content) -def update_docs(version): +def update_docs(upstream_repository, version): pattern = re.compile(r"ninja \d.\d.\d(\.[\w\-]+)*") replacement = "ninja %s" % version _update_file( @@ -221,7 +159,7 @@ def update_docs(version): pattern, replacement, verbose=False) pattern = re.compile(r"github\.com\/[\w\-_]+\/[\w\-_]+(?=\/(?:release|archive))") - replacement = "github.com/" + NINJA_RELEASES_GITHUB_REPO + replacement = "github.com/" + upstream_repository _update_file( os.path.join(ROOT_DIR, "docs/update_ninja_version.rst"), pattern, replacement, verbose=False) @@ -244,13 +182,46 @@ def update_tests(version): def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( - 'ninja_version', metavar='NINJA_VERSION', type=str, - help='CMake version of the form X.Y.Z' + "ninja_version", + metavar="NINJA_VERSION", + type=str, + help="Ninja version, shall match a tag in upstream repository", + ) + parser.add_argument( + "--upstream-repository", + metavar="UPSTREAM_REPOSITORY", + choices=["Kitware/ninja", "ninja-build/ninja"], + default="Kitware/ninja", + help="Ninja upstream repository", + ) + parser.add_argument( + "--collect-only", + action="store_true", + help="If specified, only display the archive URLs and associated hashsums", + ) + parser.add_argument( + "--quiet", + action="store_true", + help="Hide the output", ) args = parser.parse_args() - update_cmake_urls_script(args.ninja_version) - update_docs(args.ninja_version) - update_tests(args.ninja_version) + if args.collect_only: + get_ninja_archive_urls_and_sha256s(args.upstream_repository, args.ninja_version, verbose=True) + else: + update_cmake_urls_script(args.upstream_repository, args.ninja_version) + update_docs(args.upstream_repository, args.ninja_version) + update_tests(args.ninja_version) + + if not args.quiet: + msg = """\ + Complete! Now run: + + git switch -c update-to-ninja-{release} + git add -u NinjaUrls.cmake docs/index.rst README.rst tests/test_distribution.py docs/update_ninja_version.rst + git commit -m "Update to Ninja {release}" + gh pr create --fill --body "Created by update_ninja_version.py" + """ + print(textwrap.dedent(msg.format(release=args.ninja_version))) if __name__ == "__main__": diff --git a/setup.cfg b/setup.cfg index 5e3fdc5..a6e048f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [flake8] -max-line-length: 120 +max-line-length: 130 # Whether to display the pep8 instructions on failure (can be quite verbose) show-pep8: False # Whether to show source code for each failure @@ -7,7 +7,7 @@ show-source: True # Maximum cyclomatic complexity allowed max-complexity: 14 format: pylint -exclude: .git,.idea,.eggs,__pycache__,.tox,_skbuild,Ninja-src,src,versioneer.py,ninja/ninja_syntax.py,_version.py +exclude: .git,.idea,.eggs,__pycache__,.tox,_skbuild,Ninja-src,src,versioneer.py,ninja/ninja_syntax.py,_version.py,.venv,.nox [tool:pytest] testpaths = tests