NH-26173 Add sdist and wheel extension checks at package build #75
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -142,6 +142,12 @@ sdist: wrapper | |
| @python3.8 setup.py sdist | ||
| @echo -e "\nDone." | ||
|
|
||
| # Check local package source distribution archive contents, without install | ||
| CURR_DIR := $(shell pwd) | ||
| check-sdist-local: | ||
| @cd ./tests/docker/install && MODE=local APM_ROOT=$(CURR_DIR) ./_helper_check_sdist.sh | ||
| @cd $(CURR_DIR) | ||
|
|
||
| # Build the Python agent package bdist (wheels) for 64 bit many linux systems (except Alpine). | ||
| # The recipe builds the wheels for all Python versions available in the docker image EXCEPT py36, similarly to the example provided | ||
| # in the corresponding repo of the Docker images: https://github.com/pypa/manylinux#example. | ||
|
|
@@ -153,8 +159,13 @@ manylinux-wheels: wrapper | |
| @rm -rf ./tmp_dist | ||
| @echo -e "\nDone." | ||
|
|
||
| # Check local package wheel contents, without install | ||
| check-wheel-local: | ||
| @cd ./tests/docker/install && MODE=local APM_ROOT=$(CURR_DIR) ./_helper_check_wheel.sh | ||
| @cd $(CURR_DIR) | ||
|
|
||
| # Build and check the full Python agent distribution (sdist and wheels) | ||
| package: sdist check-sdist-local manylinux-wheels check-wheel-local | ||
|
There was a problem hiding this comment. Minor, the There was a problem hiding this comment. Yes good call! Addressed in 5496b3b |
||
|
|
||
| # Build the AWS lambda layer. | ||
| # temporary target directory for AWS Lambda build artifacts | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # Verifies content in the extension directory solarwinds_apm/extension. The absolute path of the directory to be verified | ||
| # needs to be passed in as the first argument of this function. As the second argument, a string containing the | ||
| # files expected under solarwinds_apm/extension needs to be provided. | ||
| echo "---- Check content of extension directory located at $1 ----" | ||
| if [ -z "$2" ]; then | ||
| echo "Failed! Files expected in the extension directory not provided." | ||
| exit 1 | ||
| fi | ||
|
|
||
| expected_files="$2" | ||
|
|
||
| pushd "$1" >/dev/null || exit 1 | ||
| found_swig_files=$(find . -not -path '.' | LC_ALL=C sort) | ||
| popd >/dev/null || exit 1 | ||
|
|
||
| if [[ ! "$found_swig_files" =~ $expected_files ]]; then | ||
| echo "FAILED! expected these files under the extension directory:" | ||
| echo "$expected_files" | ||
| echo "found:" | ||
| echo "$found_swig_files" | ||
| exit 1 | ||
| fi | ||
|
|
||
| echo -e "Content of extension directory checked successfully.\n" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,119 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # stop on error | ||
| set -e | ||
|
|
||
| # get test mode | ||
| TEST_MODES=( | ||
| "local" | ||
| "testpypi" | ||
| "packagecloud" | ||
| "pypi" | ||
| ) | ||
| if [ -z "$MODE" ] | ||
| then | ||
| echo "WARNING: Did not provide MODE for check_sdist test." | ||
| echo "Defaulting to MODE=local" | ||
| MODE=local | ||
| fi | ||
| if [[ ! " ${TEST_MODES[*]} " =~ ${MODE} ]] | ||
| then | ||
| echo "FAILED: Did not provide valid MODE for check_sdist test. Must be one of: testpypi (default), local, packagecloud, pypi." | ||
| exit 1 | ||
| else | ||
| echo "Using provided MODE=$MODE for check_sdist test." | ||
| fi | ||
|
|
||
| if [ -z "$APM_ROOT" ] | ||
| then | ||
| echo "FAILED: Did not provide valid APM_ROOT for check_sdist test." | ||
| exit 1 | ||
| fi | ||
|
|
||
|
|
||
| function get_sdist(){ | ||
| sdist_dir="$PWD/tmp/sdist" | ||
| rm -rf sdist_dir | ||
|
|
||
| if [ "$MODE" == "local" ] | ||
| then | ||
| # optionally test a previous version on local for debugging | ||
| if [ -z "$SOLARWINDS_APM_VERSION" ]; then | ||
| # no SOLARWINDS_APM_VERSION provided, thus test version of current source code | ||
| version_file=$APM_ROOT/solarwinds_apm/version.py | ||
| SOLARWINDS_APM_VERSION="$(sed -n 's/__version__ = "\(.*\)"/\1/p' "$version_file")" | ||
| echo "No SOLARWINDS_APM_VERSION provided, thus testing source code version ($SOLARWINDS_APM_VERSION)" | ||
| fi | ||
|
|
||
| sdist_tar=$APM_ROOT/dist/solarwinds_apm-${SOLARWINDS_APM_VERSION}.tar.gz | ||
| if [ ! -f "$sdist_tar" ]; then | ||
| echo "FAILED: Did not find sdist for version $SOLARWINDS_APM_VERSION. Please run 'make package' before running tests." | ||
| echo "Aborting tests." | ||
| exit 1 | ||
| fi | ||
| else | ||
| pip_options=(--no-binary solarwinds-apm --dest "$sdist_dir") | ||
| if [ "$MODE" == "testpypi" ] | ||
| then | ||
| pip_options+=(--extra-index-url https://test.pypi.org/simple/) | ||
| elif [ "$MODE" == "packagecloud" ] | ||
| then | ||
| curl -s https://packagecloud.io/install/repositories/solarwinds/solarwinds-apm-python/script.python.sh | bash | ||
| fi | ||
|
|
||
| if [ -z "$SOLARWINDS_APM_VERSION" ] | ||
| then | ||
| pip_options+=(solarwinds-apm) | ||
| else | ||
| pip_options+=(solarwinds-apm=="$SOLARWINDS_APM_VERSION") | ||
| fi | ||
|
|
||
| # shellcheck disable=SC2048 | ||
| # shellcheck disable=SC2086 | ||
| pip download ${pip_options[*]} | ||
| sdist_tar=$(find "$sdist_dir"/* -name "solarwinds_apm-*.tar.gz") | ||
| fi | ||
| } | ||
|
|
||
| function check_sdist(){ | ||
| unpack_directory="$PWD/unpack/sdist" | ||
| rm -rf "$unpack_directory" | ||
| mkdir -p "$unpack_directory" | ||
| expected_files="./VERSION | ||
| ./__init__.py | ||
| ./bson | ||
| ./bson/bson.h | ||
| ./bson/platform_hacks.h | ||
| ./liboboe-1.0-alpine-x86_64.so.0.0.0 | ||
| ./liboboe-1.0-lambda-x86_64.so.0.0.0 | ||
| ./liboboe-1.0-x86_64.so.0.0.0 | ||
| ./oboe.h | ||
| ./oboe.py | ||
| ./oboe_api.cpp | ||
| ./oboe_api.h | ||
| ./oboe_debug.h | ||
| ./oboe_wrap.cxx" | ||
| tar xzf "$1" --directory "$unpack_directory" | ||
| unpack_agent=$(find "$unpack_directory"/* -type d -name "solarwinds_apm-*") | ||
| # shellcheck disable=SC1091 | ||
| source ./_helper_check_extension_files.sh "$unpack_agent/solarwinds_apm/extension" "$expected_files" | ||
|
|
||
| if [ -z "$PIP_INSTALL" ]; then | ||
| echo -e "PIP_INSTALL not specified." | ||
| echo -e "Source distribution verified successfully.\n" | ||
| rm -rf "$unpack_directory" | ||
| exit 0 | ||
| else | ||
| echo "Installing Python agent from source" | ||
| pip install -I "$1" | ||
| fi | ||
| } | ||
|
|
||
| function get_and_check_sdist(){ | ||
| echo "#### Verifying Python agent source distribution ####" | ||
| get_sdist | ||
| check_sdist "$sdist_tar" | ||
| } | ||
|
|
||
| # start testing | ||
| get_and_check_sdist |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,131 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # stop on error | ||
| set -e | ||
|
|
||
| # get test mode | ||
| TEST_MODES=( | ||
| "local" | ||
| "testpypi" | ||
| "packagecloud" | ||
| "pypi" | ||
| ) | ||
| if [ -z "$MODE" ] | ||
| then | ||
| echo "WARNING: Did not provide MODE for check_sdist test." | ||
| echo "Defaulting to MODE=local" | ||
| MODE=local | ||
| fi | ||
| if [[ ! " ${TEST_MODES[*]} " =~ ${MODE} ]] | ||
| then | ||
| echo "FAILED: Did not provide valid MODE for check_sdist test. Must be one of: testpypi (default), local, packagecloud, pypi." | ||
| exit 1 | ||
| else | ||
| echo "Using provided MODE=$MODE for check_sdist test." | ||
| fi | ||
|
|
||
| if [ -z "$APM_ROOT" ] | ||
| then | ||
| echo "FAILED: Did not provide valid APM_ROOT for check_sdist test." | ||
| exit 1 | ||
| fi | ||
|
|
||
|
|
||
| function get_wheel(){ | ||
| wheel_dir="$PWD/tmp/wheel" | ||
| rm -rf "$wheel_dir" | ||
| if [ "$MODE" == "local" ] | ||
| then | ||
| # optionally test a previous version on local for debugging | ||
| if [ -z "$SOLARWINDS_APM_VERSION" ]; then | ||
| # no SOLARWINDS_APM_VERSION provided, thus test version of current source code | ||
| version_file=$APM_ROOT/solarwinds_apm/version.py | ||
| SOLARWINDS_APM_VERSION="$(sed -n 's/__version__ = "\(.*\)"/\1/p' "$version_file")" | ||
| echo "No SOLARWINDS_APM_VERSION provided, thus testing source code version ($SOLARWINDS_APM_VERSION)" | ||
| fi | ||
|
|
||
| if [ -z "$PIP_INSTALL" ]; then | ||
| echo -e "PIP_INSTALL not specified." | ||
| echo -e "Only testing the cp38 x86_64 wheel under ${APM_ROOT}" | ||
| tested_wheel=$(find "$APM_ROOT"/dist/* -name "solarwinds_apm-$SOLARWINDS_APM_VERSION-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl") | ||
| else | ||
| # we need to select the right wheel (there might be multiple wheel versions in the dist directory) | ||
| pip download \ | ||
| --only-binary solarwinds_apm \ | ||
| --find-links "$APM_ROOT"/dist \ | ||
| --no-index \ | ||
| --dest "$wheel_dir" \ | ||
| --no-deps \ | ||
| solarwinds_apm=="$SOLARWINDS_APM_VERSION" | ||
| tested_wheel=$(find "$wheel_dir"/* -name "solarwinds_apm-$SOLARWINDS_APM_VERSION*.*.whl") | ||
| fi | ||
|
|
||
| if [ ! -f "$tested_wheel" ]; then | ||
| echo "FAILED: Did not find wheel for version $SOLARWINDS_APM_VERSION. Please run 'make package' before running tests." | ||
| echo "Aborting tests." | ||
| exit 1 | ||
| fi | ||
| else | ||
| pip_options=(--only-binary solarwinds-apm --dest "$wheel_dir") | ||
| if [ "$MODE" == "testpypi" ] | ||
| then | ||
| pip_options+=(--extra-index-url https://test.pypi.org/simple/) | ||
| elif [ "$MODE" == "packagecloud" ] | ||
| then | ||
| curl -s https://packagecloud.io/install/repositories/solarwinds/solarwinds-apm-python/script.python.sh | bash | ||
| fi | ||
|
|
||
| if [ -z "$SOLARWINDS_APM_VERSION" ] | ||
| then | ||
| pip_options+=(solarwinds-apm) | ||
| else | ||
| pip_options+=(solarwinds-apm=="$SOLARWINDS_APM_VERSION") | ||
| fi | ||
|
|
||
| # shellcheck disable=SC2048 | ||
| # shellcheck disable=SC2086 | ||
| pip download ${pip_options[*]} | ||
| tested_wheel=$(find "$wheel_dir"/* -name "solarwinds_apm-*.*.whl") | ||
| fi | ||
| } | ||
|
|
||
| function check_wheel(){ | ||
| unpack_directory="$PWD/unpack/wheel" | ||
| rm -rf "$unpack_directory" | ||
| mkdir -p "$unpack_directory" | ||
| expected_files="./VERSION | ||
| ./__init__.py | ||
| ./_oboe.*.so | ||
| ./bson | ||
| ./bson/bson.h | ||
| ./bson/platform_hacks.h | ||
| ./liboboe-1.0.so.0 | ||
| ./oboe.py" | ||
| unzip "$tested_wheel" -d "$unpack_directory" | ||
| # shellcheck disable=SC1091 | ||
| source ./_helper_check_extension_files.sh "$unpack_directory/solarwinds_apm/extension" "$expected_files" | ||
|
|
||
| if [ -z "$PIP_INSTALL" ]; then | ||
| echo -e "PIP_INSTALL not specified." | ||
| echo -e "Python wheel verified successfully.\n" | ||
| rm -rf "$unpack_directory" | ||
| exit 0 | ||
| else | ||
| echo "Installing Python agent from wheel" | ||
| pip install -I "$tested_wheel" | ||
| fi | ||
| } | ||
|
|
||
| function get_and_check_wheel(){ | ||
| echo "#### Verifying Python agent wheel distribution ####" | ||
| # Python wheels are not available under Alpine Linux | ||
| if [[ -f /etc/os-release && "$(cat /etc/os-release)" =~ "Alpine" ]]; then | ||
| echo "Wheels are not available on Alpine Linux, skip wheel tests." | ||
| else | ||
| get_wheel | ||
| check_wheel "$tested_wheel" | ||
| fi | ||
| } | ||
|
|
||
| # start testing | ||
| get_and_check_wheel |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Nice!