From 4664eb10d158e28e144fe9be243ae413b6e799f5 Mon Sep 17 00:00:00 2001 From: Sevag H Date: Wed, 16 Nov 2022 12:32:18 -0500 Subject: [PATCH] Feat/pip wheel version script (#28) * Add pip wheel version script * add debugging to pip version script * Rewrite rapids-twine in bash * Address shellcheck suggestions * Print debug statements to stderr * Simplify rapids-pip-wheel-version * Pass epoch timestamp into pip-version script * Use set -u instead of rapids-require-env-var --- tools/rapids-pip-wheel-version | 34 +++++++++++++++ tools/rapids-twine | 34 +++++++++++++++ tools/rapids-twine.py | 80 ---------------------------------- 3 files changed, 68 insertions(+), 80 deletions(-) create mode 100755 tools/rapids-pip-wheel-version create mode 100755 tools/rapids-twine delete mode 100755 tools/rapids-twine.py diff --git a/tools/rapids-pip-wheel-version b/tools/rapids-pip-wheel-version new file mode 100755 index 0000000..3e8431b --- /dev/null +++ b/tools/rapids-pip-wheel-version @@ -0,0 +1,34 @@ +#!/bin/bash +# A utility script that generates a pip wheel version +# Positional Arguments: +# 1) epoch timestamp +set -exo pipefail +export RAPIDS_SCRIPT_NAME="rapids-pip-wheel-version" + +if [ -z "$1" ]; then + rapids-echo-stderr "Must specify input arguments: EPOCH_TIMESTAMP" + exit 1 +fi +epoch_timestamp="$1" + +function distutilsNormalizeVersion { + echo -n "$(python3 -c "from setuptools.extern import packaging; print(packaging.version.Version('$1'))")" +} + +rapids-echo-stderr "pwd is $(pwd)" + +latest_git_tag="$(git describe --tag --abbrev=0)" + +# drop letters ('v', 'a', etc.) from tag +# v22.12.00a -> 22.12.00 +# v22.12.00 -> 22.12.00 + +latest_git_tag="${latest_git_tag//[a-z]/}" + +# normalize with distutils logic +latest_git_tag="$(distutilsNormalizeVersion "${latest_git_tag}")" + +# add epoch timestamp so more recent wheels are installed first +versioneer_override="$latest_git_tag.$epoch_timestamp" + +echo -n "${versioneer_override}" diff --git a/tools/rapids-twine b/tools/rapids-twine new file mode 100755 index 0000000..5b5654c --- /dev/null +++ b/tools/rapids-twine @@ -0,0 +1,34 @@ +#!/bin/bash +# A utility script that wraps twine to support build tags (PEP427) +# +# Positional Arguments: +# 1) wheel dir +set -exou pipefail +export RAPIDS_SCRIPT_NAME="rapids-twine" + +if [ -z "$1" ]; then + rapids-echo-stderr "Must specify input arguments: WHEEL_DIR" + exit 1 +fi +wheeldir="$1" + +build_tag="${RAPIDS_PY_WHEEL_BUILD_TAG:-""}" + +if [ "${build_tag}" != "" ]; then + echo "Checking if build tag ${build_tag} is legal..." + if [[ "${build_tag}" =~ [^[:digit:]] ]]; then + rapids-echo-stderr "Build tag can only be digits" + exit 1 + fi + + echo "Need to apply build tag ${build_tag}..." + for wheelfile in "${wheeldir}"/*.whl; do + wheel_version=$(echo "$wheelfile" | cut -d'-' -f2) + replacement_wheel_version="${wheel_version}-${build_tag}" + wheelnewfile="${wheelfile/$wheel_version/$replacement_wheel_version}" + + mv "${wheelfile}" "${wheelnewfile}" + done +fi + +twine upload --disable-progress-bar --non-interactive "${wheeldir}"/*.whl diff --git a/tools/rapids-twine.py b/tools/rapids-twine.py deleted file mode 100755 index b9a167d..0000000 --- a/tools/rapids-twine.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python3 -# A utility script that uploads wheels to pip indexes with twine to support build numbers on conflict -# See: -# - https://stackoverflow.com/a/63944201 -# - https://peps.python.org/pep-0427/, section on 'build tag' -# -# Positional Arguments: -# 1) directory containing wheels - -import subprocess -import os -import sys -import shutil -import tempfile -from setuptools.extern import packaging - -echo_prefix=" [rapids-twine.py] " -rapids_echo_stderr_fn=lambda x: print(x, file=sys.stderr) -twine_upload_args = ["--disable-progress-bar", "--non-interactive"] - - -def _twine_upload(wheel): - print(f"{echo_prefix}Working on {wheel=}") - twine_out = subprocess.run( - ["twine", "upload", *twine_upload_args, wheel], - capture_output=True - ) - retcode, stdout = twine_out.returncode, twine_out.stdout - - print(f"{echo_prefix}Twine output: '{stdout}'") - return retcode == 0 - - -if __name__ == '__main__': - wheel_dir = None - try: - wheel_dir = sys.argv[1] - except IndexError: - rapids_echo_stderr_fn(f"{echo_prefix}Must specify input argument: WHEEL_DIR") - sys.exit(1) - - if not os.path.isdir(wheel_dir): - rapids_echo_stderr_fn(f"{echo_prefix}Path '{wheel_dir}' is not a directory") - - wheel_base_version = os.environ["RAPIDS_PY_WHEEL_VERSIONEER_OVERRIDE"] - build_tag = os.getenv("RAPIDS_PY_WHEEL_BUILD_TAG", default="") - - # replace override with pypi-normalized string e.g. 22.10.00a --> 22.10.0a0 - wheel_base_version_normalized = str(packaging.version.Version(wheel_base_version)) - - rapids_echo_stderr_fn(f"{echo_prefix}Normalizing '{wheel_base_version}' to '{wheel_base_version_normalized} to match setuptools...") - wheel_base_version = wheel_base_version_normalized - - for wheel_file_name in os.listdir(wheel_dir): - wheel_file_path = os.path.join(wheel_dir, wheel_file_name) - - if build_tag == '': - success = _twine_upload(wheel_file_path) - if success: - continue - else: - sys.exit(1) - else: - rapids_echo_stderr_fn(f"{echo_prefix}Using specified build tag '{build_tag}'") - wheel_next_version = f"{wheel_base_version}-{build_tag}" - - # copy original wheels to a tempdir - with tempfile.TemporaryDirectory() as tempdir: - rapids_echo_stderr_fn(f"{echo_prefix}Replacing '{wheel_base_version}' with '{wheel_next_version}'") - wheel_next_file_name = wheel_file_name.replace(wheel_base_version, wheel_next_version) - wheel_next_file_path = os.path.join(tempdir, wheel_next_file_name) - - rapids_echo_stderr_fn(f"{echo_prefix}Copying {wheel_file_path} to {wheel_next_file_path}") - shutil.copy(wheel_file_path, wheel_next_file_path) - - next_success = _twine_upload(wheel_next_file_path) - if next_success: - continue - else: - sys.exit(1)