From 587ba06ca63dbd81d59c0178e91890d83e41a665 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Wed, 7 Apr 2021 14:35:10 -0300 Subject: [PATCH 01/15] zarr support --- recipe/bld.bat | 3 +++ recipe/build.sh | 6 ++++++ recipe/meta.yaml | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/recipe/bld.bat b/recipe/bld.bat index 95720f67..86d177d9 100644 --- a/recipe/bld.bat +++ b/recipe/bld.bat @@ -19,6 +19,9 @@ cmake -LAH -G "NMake Makefiles" ^ -DHDF5_HL_LIBRARY="%LIBRARY_LIB:\=/%/hdf5_hl.lib" ^ -DHDF5_INCLUDE_DIR="%LIBRARY_INC:\=/%" ^ -DCMAKE_C_FLAGS="-DH5_BUILT_AS_DYNAMIC_LIB" ^ + -DENABLE_NCZARR=on ^ + -DENABLE_NCZARR_S3=off ^ + -DENABLE_NCZARR_S3_TESTS=off ^ %SRC_DIR% if errorlevel 1 exit \b 1 diff --git a/recipe/build.sh b/recipe/build.sh index 42b5f7d1..957cb3f0 100644 --- a/recipe/build.sh +++ b/recipe/build.sh @@ -78,6 +78,9 @@ cmake -DCMAKE_INSTALL_PREFIX=${PREFIX} \ -DENABLE_BYTERANGE=ON \ ${CMAKE_PLATFORM_FLAGS[@]} \ ${PARALLEL} \ + -DENABLE_NCZARR=on \ + -DENABLE_NCZARR_S3=off \ + -DENABLE_NCZARR_S3_TESTS=off \ ${SRC_DIR} # ctest # Run only for the shared lib build to save time. make install -j${CPU_COUNT} ${VERBOSE_CM} @@ -100,6 +103,9 @@ cmake -DCMAKE_INSTALL_PREFIX=${PREFIX} \ -DENABLE_CDF5=ON \ ${CMAKE_PLATFORM_FLAGS[@]} \ ${PARALLEL} \ + -DENABLE_NCZARR=on \ + -DENABLE_NCZARR_S3=off \ + -DENABLE_NCZARR_S3_TESTS=off \ ${SRC_DIR} make install -j${CPU_COUNT} ${VERBOSE_CM} ctest -VV --output-on-failure -j${CPU_COUNT} diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 2713eb42..37113010 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -1,5 +1,5 @@ {% set version = "4.8.0" %} -{% set build = 0 %} +{% set build = 1 %} # recipe-lint fails if mpi is undefined {% set mpi = mpi or 'nompi' %} From 490da70c255681ba32fab909b18704a69ae16fb8 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Wed, 7 Apr 2021 19:55:17 -0300 Subject: [PATCH 02/15] add zarr,zip support --- recipe/meta.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 37113010..051fc92b 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -76,6 +76,7 @@ requirements: - hdf5 - hdf5 * {{ mpi_prefix }}_* - jpeg + - libzip run: - {{ mpi }} # [mpi != 'nompi'] - bzip2 @@ -84,6 +85,7 @@ requirements: - hdf4 # [not aarch64] - hdf5 * {{ mpi_prefix }}_* - jpeg + - libzip # Curl should be pulling this... Investigate it later. - libssh2 # [win] From 39487bc8023e2e13c93cdfb37e2a47b214ef8a2d Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Wed, 7 Apr 2021 19:55:46 -0300 Subject: [PATCH 03/15] MNT: Re-rendered with conda-build 3.21.4, conda-smithy 3.10.0, and conda-forge-pinning 2021.04.07.18.46.00 --- .scripts/build_steps.sh | 27 ++++++++++++++++----------- .scripts/logging_utils.sh | 9 +++++++-- .scripts/run_docker_build.sh | 13 +++++++++---- .scripts/run_osx_build.sh | 31 ++++++++++++++++++------------- README.md | 1 + 5 files changed, 51 insertions(+), 30 deletions(-) diff --git a/.scripts/build_steps.sh b/.scripts/build_steps.sh index 522eabcf..71a06860 100755 --- a/.scripts/build_steps.sh +++ b/.scripts/build_steps.sh @@ -10,9 +10,10 @@ export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/feedstock_root}" source ${FEEDSTOCK_ROOT}/.scripts/logging_utils.sh -endgroup "Start Docker" +( endgroup "Start Docker" ) 2> /dev/null + +( startgroup "Configuring conda" ) 2> /dev/null -startgroup "Configuring conda" export PYTHONUNBUFFERED=1 export RECIPE_ROOT="${RECIPE_ROOT:-/home/conda/recipe_root}" export CI_SUPPORT="${FEEDSTOCK_ROOT}/.ci_support" @@ -36,34 +37,38 @@ source run_conda_forge_build_setup # make the build number clobber make_build_number "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" -endgroup "Configuring conda" + +( endgroup "Configuring conda" ) 2> /dev/null if [[ "${BUILD_WITH_CONDA_DEBUG:-0}" == 1 ]]; then - startgroup "Running conda debug" if [[ "x${BUILD_OUTPUT_ID:-}" != "x" ]]; then EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --output-id ${BUILD_OUTPUT_ID}" fi conda debug "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \ ${EXTRA_CB_OPTIONS:-} \ --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" - endgroup "Running conda debug" + # Drop into an interactive shell /bin/bash else - startgroup "Running conda $BUILD_CMD" conda $BUILD_CMD "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \ --suppress-variables ${EXTRA_CB_OPTIONS:-} \ --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" - endgroup "Running conda build" - startgroup "Validating outputs" + ( startgroup "Validating outputs" ) 2> /dev/null + validate_recipe_outputs "${FEEDSTOCK_NAME}" - endgroup "Validating outputs" + + ( endgroup "Validating outputs" ) 2> /dev/null + + ( startgroup "Uploading packages" ) 2> /dev/null if [[ "${UPLOAD_PACKAGES}" != "False" ]]; then - startgroup "Uploading packages" upload_package --validate --feedstock-name="${FEEDSTOCK_NAME}" "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" - endgroup "Uploading packages" fi + + ( endgroup "Uploading packages" ) 2> /dev/null fi +( startgroup "Final checks" ) 2> /dev/null + touch "${FEEDSTOCK_ROOT}/build_artifacts/conda-forge-build-done-${CONFIG}" \ No newline at end of file diff --git a/.scripts/logging_utils.sh b/.scripts/logging_utils.sh index a53ef3f2..57bc95c2 100644 --- a/.scripts/logging_utils.sh +++ b/.scripts/logging_utils.sh @@ -13,18 +13,23 @@ function startgroup { travis ) echo "$1" echo -en 'travis_fold:start:'"${1// /}"'\\r';; + github_actions ) + echo "::group::$1";; * ) echo "$1";; esac -} +} 2> /dev/null function endgroup { # End a foldable group of log lines # Pass a single argument, quoted + case ${CI:-} in azure ) echo "##[endgroup]";; travis ) echo -en 'travis_fold:end:'"${1// /}"'\\r';; + github_actions ) + echo "::endgroup::";; esac -} +} 2> /dev/null diff --git a/.scripts/run_docker_build.sh b/.scripts/run_docker_build.sh index 16d6c15a..44ad3ce0 100755 --- a/.scripts/run_docker_build.sh +++ b/.scripts/run_docker_build.sh @@ -7,7 +7,7 @@ source .scripts/logging_utils.sh -startgroup "Configure Docker" +( startgroup "Configure Docker" ) 2> /dev/null set -xeo pipefail @@ -69,9 +69,11 @@ DOCKER_RUN_ARGS="${CONDA_FORGE_DOCKER_RUN_ARGS}" if [ -z "${CI}" ]; then DOCKER_RUN_ARGS="-it ${DOCKER_RUN_ARGS}" fi -endgroup "Configure Docker" -startgroup "Start Docker" +( endgroup "Configure Docker" ) 2> /dev/null + +( startgroup "Start Docker" ) 2> /dev/null + export UPLOAD_PACKAGES="${UPLOAD_PACKAGES:-True}" docker run ${DOCKER_RUN_ARGS} \ -v "${RECIPE_ROOT}":/home/conda/recipe_root:rw,z,delegated \ @@ -94,4 +96,7 @@ docker run ${DOCKER_RUN_ARGS} \ /home/conda/feedstock_root/${PROVIDER_DIR}/build_steps.sh # verify that the end of the script was reached -test -f "$DONE_CANARY" \ No newline at end of file +test -f "$DONE_CANARY" + +# This closes the last group opened in `build_steps.sh` +( endgroup "Final checks" ) 2> /dev/null \ No newline at end of file diff --git a/.scripts/run_osx_build.sh b/.scripts/run_osx_build.sh index 614ca844..39436ecb 100755 --- a/.scripts/run_osx_build.sh +++ b/.scripts/run_osx_build.sh @@ -2,16 +2,19 @@ source .scripts/logging_utils.sh -set -x +set -xe + +( startgroup "Installing a fresh version of Miniforge" ) 2> /dev/null -startgroup "Installing a fresh version of Miniforge" MINIFORGE_URL="https://github.com/conda-forge/miniforge/releases/latest/download" MINIFORGE_FILE="Miniforge3-MacOSX-x86_64.sh" curl -L -O "${MINIFORGE_URL}/${MINIFORGE_FILE}" bash $MINIFORGE_FILE -b -endgroup "Installing a fresh version of Miniforge" -startgroup "Configuring conda" +( endgroup "Installing a fresh version of Miniforge" ) 2> /dev/null + +( startgroup "Configuring conda" ) 2> /dev/null + BUILD_CMD=build source ${HOME}/miniforge3/etc/profile.d/conda.sh @@ -34,22 +37,24 @@ echo -e "\n\nRunning the build setup script." source run_conda_forge_build_setup -endgroup "Configuring conda" -set -e +( endgroup "Configuring conda" ) 2> /dev/null + -startgroup "Running conda $BUILD_CMD" echo -e "\n\nMaking the build clobber file" make_build_number ./ ./recipe ./.ci_support/${CONFIG}.yaml conda $BUILD_CMD ./recipe -m ./.ci_support/${CONFIG}.yaml --suppress-variables --clobber-file ./.ci_support/clobber_${CONFIG}.yaml ${EXTRA_CB_OPTIONS:-} -endgroup "Running conda build" -startgroup "Validating outputs" +( startgroup "Validating outputs" ) 2> /dev/null + validate_recipe_outputs "${FEEDSTOCK_NAME}" -endgroup "Validating outputs" + +( endgroup "Validating outputs" ) 2> /dev/null + +( startgroup "Uploading packages" ) 2> /dev/null if [[ "${UPLOAD_PACKAGES}" != "False" ]]; then - startgroup "Uploading packages" upload_package --validate --feedstock-name="${FEEDSTOCK_NAME}" ./ ./recipe ./.ci_support/${CONFIG}.yaml - endgroup "Uploading packages" -fi \ No newline at end of file +fi + +( endgroup "Uploading packages" ) 2> /dev/null \ No newline at end of file diff --git a/README.md b/README.md index 5d46a20e..212393de 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,7 @@ Installing `libnetcdf` from the `conda-forge` channel can be achieved by adding ``` conda config --add channels conda-forge +conda config --set channel_priority strict ``` Once the `conda-forge` channel has been enabled, `libnetcdf` can be installed with: From 72194b7097241ea06af61e4e9f10e6e0383e428c Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Fri, 9 Apr 2021 14:52:33 -0300 Subject: [PATCH 04/15] MNT: Re-rendered with conda-build 3.21.4, conda-smithy 3.10.0, and conda-forge-pinning 2021.04.09.17.43.25 --- .scripts/build_steps.sh | 27 ++++++++++++++++----------- .scripts/logging_utils.sh | 9 +++++++-- .scripts/run_docker_build.sh | 13 +++++++++---- .scripts/run_osx_build.sh | 31 ++++++++++++++++++------------- README.md | 1 + 5 files changed, 51 insertions(+), 30 deletions(-) diff --git a/.scripts/build_steps.sh b/.scripts/build_steps.sh index 522eabcf..71a06860 100755 --- a/.scripts/build_steps.sh +++ b/.scripts/build_steps.sh @@ -10,9 +10,10 @@ export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/feedstock_root}" source ${FEEDSTOCK_ROOT}/.scripts/logging_utils.sh -endgroup "Start Docker" +( endgroup "Start Docker" ) 2> /dev/null + +( startgroup "Configuring conda" ) 2> /dev/null -startgroup "Configuring conda" export PYTHONUNBUFFERED=1 export RECIPE_ROOT="${RECIPE_ROOT:-/home/conda/recipe_root}" export CI_SUPPORT="${FEEDSTOCK_ROOT}/.ci_support" @@ -36,34 +37,38 @@ source run_conda_forge_build_setup # make the build number clobber make_build_number "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" -endgroup "Configuring conda" + +( endgroup "Configuring conda" ) 2> /dev/null if [[ "${BUILD_WITH_CONDA_DEBUG:-0}" == 1 ]]; then - startgroup "Running conda debug" if [[ "x${BUILD_OUTPUT_ID:-}" != "x" ]]; then EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --output-id ${BUILD_OUTPUT_ID}" fi conda debug "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \ ${EXTRA_CB_OPTIONS:-} \ --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" - endgroup "Running conda debug" + # Drop into an interactive shell /bin/bash else - startgroup "Running conda $BUILD_CMD" conda $BUILD_CMD "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \ --suppress-variables ${EXTRA_CB_OPTIONS:-} \ --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" - endgroup "Running conda build" - startgroup "Validating outputs" + ( startgroup "Validating outputs" ) 2> /dev/null + validate_recipe_outputs "${FEEDSTOCK_NAME}" - endgroup "Validating outputs" + + ( endgroup "Validating outputs" ) 2> /dev/null + + ( startgroup "Uploading packages" ) 2> /dev/null if [[ "${UPLOAD_PACKAGES}" != "False" ]]; then - startgroup "Uploading packages" upload_package --validate --feedstock-name="${FEEDSTOCK_NAME}" "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" - endgroup "Uploading packages" fi + + ( endgroup "Uploading packages" ) 2> /dev/null fi +( startgroup "Final checks" ) 2> /dev/null + touch "${FEEDSTOCK_ROOT}/build_artifacts/conda-forge-build-done-${CONFIG}" \ No newline at end of file diff --git a/.scripts/logging_utils.sh b/.scripts/logging_utils.sh index a53ef3f2..57bc95c2 100644 --- a/.scripts/logging_utils.sh +++ b/.scripts/logging_utils.sh @@ -13,18 +13,23 @@ function startgroup { travis ) echo "$1" echo -en 'travis_fold:start:'"${1// /}"'\\r';; + github_actions ) + echo "::group::$1";; * ) echo "$1";; esac -} +} 2> /dev/null function endgroup { # End a foldable group of log lines # Pass a single argument, quoted + case ${CI:-} in azure ) echo "##[endgroup]";; travis ) echo -en 'travis_fold:end:'"${1// /}"'\\r';; + github_actions ) + echo "::endgroup::";; esac -} +} 2> /dev/null diff --git a/.scripts/run_docker_build.sh b/.scripts/run_docker_build.sh index 16d6c15a..44ad3ce0 100755 --- a/.scripts/run_docker_build.sh +++ b/.scripts/run_docker_build.sh @@ -7,7 +7,7 @@ source .scripts/logging_utils.sh -startgroup "Configure Docker" +( startgroup "Configure Docker" ) 2> /dev/null set -xeo pipefail @@ -69,9 +69,11 @@ DOCKER_RUN_ARGS="${CONDA_FORGE_DOCKER_RUN_ARGS}" if [ -z "${CI}" ]; then DOCKER_RUN_ARGS="-it ${DOCKER_RUN_ARGS}" fi -endgroup "Configure Docker" -startgroup "Start Docker" +( endgroup "Configure Docker" ) 2> /dev/null + +( startgroup "Start Docker" ) 2> /dev/null + export UPLOAD_PACKAGES="${UPLOAD_PACKAGES:-True}" docker run ${DOCKER_RUN_ARGS} \ -v "${RECIPE_ROOT}":/home/conda/recipe_root:rw,z,delegated \ @@ -94,4 +96,7 @@ docker run ${DOCKER_RUN_ARGS} \ /home/conda/feedstock_root/${PROVIDER_DIR}/build_steps.sh # verify that the end of the script was reached -test -f "$DONE_CANARY" \ No newline at end of file +test -f "$DONE_CANARY" + +# This closes the last group opened in `build_steps.sh` +( endgroup "Final checks" ) 2> /dev/null \ No newline at end of file diff --git a/.scripts/run_osx_build.sh b/.scripts/run_osx_build.sh index 614ca844..39436ecb 100755 --- a/.scripts/run_osx_build.sh +++ b/.scripts/run_osx_build.sh @@ -2,16 +2,19 @@ source .scripts/logging_utils.sh -set -x +set -xe + +( startgroup "Installing a fresh version of Miniforge" ) 2> /dev/null -startgroup "Installing a fresh version of Miniforge" MINIFORGE_URL="https://github.com/conda-forge/miniforge/releases/latest/download" MINIFORGE_FILE="Miniforge3-MacOSX-x86_64.sh" curl -L -O "${MINIFORGE_URL}/${MINIFORGE_FILE}" bash $MINIFORGE_FILE -b -endgroup "Installing a fresh version of Miniforge" -startgroup "Configuring conda" +( endgroup "Installing a fresh version of Miniforge" ) 2> /dev/null + +( startgroup "Configuring conda" ) 2> /dev/null + BUILD_CMD=build source ${HOME}/miniforge3/etc/profile.d/conda.sh @@ -34,22 +37,24 @@ echo -e "\n\nRunning the build setup script." source run_conda_forge_build_setup -endgroup "Configuring conda" -set -e +( endgroup "Configuring conda" ) 2> /dev/null + -startgroup "Running conda $BUILD_CMD" echo -e "\n\nMaking the build clobber file" make_build_number ./ ./recipe ./.ci_support/${CONFIG}.yaml conda $BUILD_CMD ./recipe -m ./.ci_support/${CONFIG}.yaml --suppress-variables --clobber-file ./.ci_support/clobber_${CONFIG}.yaml ${EXTRA_CB_OPTIONS:-} -endgroup "Running conda build" -startgroup "Validating outputs" +( startgroup "Validating outputs" ) 2> /dev/null + validate_recipe_outputs "${FEEDSTOCK_NAME}" -endgroup "Validating outputs" + +( endgroup "Validating outputs" ) 2> /dev/null + +( startgroup "Uploading packages" ) 2> /dev/null if [[ "${UPLOAD_PACKAGES}" != "False" ]]; then - startgroup "Uploading packages" upload_package --validate --feedstock-name="${FEEDSTOCK_NAME}" ./ ./recipe ./.ci_support/${CONFIG}.yaml - endgroup "Uploading packages" -fi \ No newline at end of file +fi + +( endgroup "Uploading packages" ) 2> /dev/null \ No newline at end of file diff --git a/README.md b/README.md index 5d46a20e..212393de 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,7 @@ Installing `libnetcdf` from the `conda-forge` channel can be achieved by adding ``` conda config --add channels conda-forge +conda config --set channel_priority strict ``` Once the `conda-forge` channel has been enabled, `libnetcdf` can be installed with: From b054309580e7e68847e58d441039d638a7cc395a Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Fri, 9 Apr 2021 14:54:22 -0300 Subject: [PATCH 05/15] skip tests on ppc --- recipe/build.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/recipe/build.sh b/recipe/build.sh index 957cb3f0..cfc2d7f0 100644 --- a/recipe/build.sh +++ b/recipe/build.sh @@ -110,6 +110,13 @@ cmake -DCMAKE_INSTALL_PREFIX=${PREFIX} \ make install -j${CPU_COUNT} ${VERBOSE_CM} ctest -VV --output-on-failure -j${CPU_COUNT} +# Skip failing test on ppc74le +if [ ${target_platform} == "linux-ppc64le" ]; then + SKIP="-E nc_test" +fi + +ctest -VV --output-on-failure -j${CPU_COUNT} "${SKIP}" + if [[ ${c_compiler} != "toolchain_c" ]]; then # Fix build paths in cmake artifacts for fname in `ls ${PREFIX}/lib/cmake/netCDF/*`; do From 9529d6cc9ae453f446f790e6fc3890bac3c33d9a Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Fri, 9 Apr 2021 15:06:03 -0300 Subject: [PATCH 06/15] MNT: Re-rendered with conda-build 3.21.4, conda-smithy 3.10.0, and conda-forge-pinning 2021.04.09.17.43.25 --- .ci_support/linux_64_mpimpich.yaml | 2 ++ .ci_support/linux_64_mpinompi.yaml | 2 ++ .ci_support/linux_64_mpiopenmpi.yaml | 2 ++ .ci_support/linux_aarch64_mpimpich.yaml | 2 ++ .ci_support/linux_aarch64_mpinompi.yaml | 2 ++ .ci_support/linux_aarch64_mpiopenmpi.yaml | 2 ++ .ci_support/linux_ppc64le_mpimpich.yaml | 2 ++ .ci_support/linux_ppc64le_mpinompi.yaml | 2 ++ .ci_support/linux_ppc64le_mpiopenmpi.yaml | 2 ++ .ci_support/osx_64_mpimpich.yaml | 2 ++ .ci_support/osx_64_mpinompi.yaml | 2 ++ .ci_support/osx_64_mpiopenmpi.yaml | 2 ++ .ci_support/win_64_.yaml | 2 ++ 13 files changed, 26 insertions(+) diff --git a/.ci_support/linux_64_mpimpich.yaml b/.ci_support/linux_64_mpimpich.yaml index d1c5bbdb..029cba6b 100644 --- a/.ci_support/linux_64_mpimpich.yaml +++ b/.ci_support/linux_64_mpimpich.yaml @@ -24,6 +24,8 @@ hdf5: - 1.10.6 jpeg: - '9' +libzip: +- '1' mpi: - mpich pin_run_as_build: diff --git a/.ci_support/linux_64_mpinompi.yaml b/.ci_support/linux_64_mpinompi.yaml index 9a8157c6..a062cc32 100644 --- a/.ci_support/linux_64_mpinompi.yaml +++ b/.ci_support/linux_64_mpinompi.yaml @@ -24,6 +24,8 @@ hdf5: - 1.10.6 jpeg: - '9' +libzip: +- '1' mpi: - nompi pin_run_as_build: diff --git a/.ci_support/linux_64_mpiopenmpi.yaml b/.ci_support/linux_64_mpiopenmpi.yaml index f46fe52b..05275e9b 100644 --- a/.ci_support/linux_64_mpiopenmpi.yaml +++ b/.ci_support/linux_64_mpiopenmpi.yaml @@ -24,6 +24,8 @@ hdf5: - 1.10.6 jpeg: - '9' +libzip: +- '1' mpi: - openmpi pin_run_as_build: diff --git a/.ci_support/linux_aarch64_mpimpich.yaml b/.ci_support/linux_aarch64_mpimpich.yaml index e8d261d6..7edf1463 100644 --- a/.ci_support/linux_aarch64_mpimpich.yaml +++ b/.ci_support/linux_aarch64_mpimpich.yaml @@ -26,6 +26,8 @@ hdf5: - 1.10.6 jpeg: - '9' +libzip: +- '1' mpi: - mpich pin_run_as_build: diff --git a/.ci_support/linux_aarch64_mpinompi.yaml b/.ci_support/linux_aarch64_mpinompi.yaml index 8597ea97..6ef50659 100644 --- a/.ci_support/linux_aarch64_mpinompi.yaml +++ b/.ci_support/linux_aarch64_mpinompi.yaml @@ -26,6 +26,8 @@ hdf5: - 1.10.6 jpeg: - '9' +libzip: +- '1' mpi: - nompi pin_run_as_build: diff --git a/.ci_support/linux_aarch64_mpiopenmpi.yaml b/.ci_support/linux_aarch64_mpiopenmpi.yaml index 6c443266..131cb69d 100644 --- a/.ci_support/linux_aarch64_mpiopenmpi.yaml +++ b/.ci_support/linux_aarch64_mpiopenmpi.yaml @@ -26,6 +26,8 @@ hdf5: - 1.10.6 jpeg: - '9' +libzip: +- '1' mpi: - openmpi pin_run_as_build: diff --git a/.ci_support/linux_ppc64le_mpimpich.yaml b/.ci_support/linux_ppc64le_mpimpich.yaml index af478b41..3b43dcd4 100644 --- a/.ci_support/linux_ppc64le_mpimpich.yaml +++ b/.ci_support/linux_ppc64le_mpimpich.yaml @@ -24,6 +24,8 @@ hdf5: - 1.10.6 jpeg: - '9' +libzip: +- '1' mpi: - mpich pin_run_as_build: diff --git a/.ci_support/linux_ppc64le_mpinompi.yaml b/.ci_support/linux_ppc64le_mpinompi.yaml index 233f3826..91534466 100644 --- a/.ci_support/linux_ppc64le_mpinompi.yaml +++ b/.ci_support/linux_ppc64le_mpinompi.yaml @@ -24,6 +24,8 @@ hdf5: - 1.10.6 jpeg: - '9' +libzip: +- '1' mpi: - nompi pin_run_as_build: diff --git a/.ci_support/linux_ppc64le_mpiopenmpi.yaml b/.ci_support/linux_ppc64le_mpiopenmpi.yaml index 44b9b74b..0cc26f15 100644 --- a/.ci_support/linux_ppc64le_mpiopenmpi.yaml +++ b/.ci_support/linux_ppc64le_mpiopenmpi.yaml @@ -24,6 +24,8 @@ hdf5: - 1.10.6 jpeg: - '9' +libzip: +- '1' mpi: - openmpi pin_run_as_build: diff --git a/.ci_support/osx_64_mpimpich.yaml b/.ci_support/osx_64_mpimpich.yaml index 8578b299..af5208d2 100644 --- a/.ci_support/osx_64_mpimpich.yaml +++ b/.ci_support/osx_64_mpimpich.yaml @@ -22,6 +22,8 @@ hdf5: - 1.10.6 jpeg: - '9' +libzip: +- '1' macos_machine: - x86_64-apple-darwin13.4.0 mpi: diff --git a/.ci_support/osx_64_mpinompi.yaml b/.ci_support/osx_64_mpinompi.yaml index 5822fd5d..335204c4 100644 --- a/.ci_support/osx_64_mpinompi.yaml +++ b/.ci_support/osx_64_mpinompi.yaml @@ -22,6 +22,8 @@ hdf5: - 1.10.6 jpeg: - '9' +libzip: +- '1' macos_machine: - x86_64-apple-darwin13.4.0 mpi: diff --git a/.ci_support/osx_64_mpiopenmpi.yaml b/.ci_support/osx_64_mpiopenmpi.yaml index 3a5c14c4..acf2da1f 100644 --- a/.ci_support/osx_64_mpiopenmpi.yaml +++ b/.ci_support/osx_64_mpiopenmpi.yaml @@ -22,6 +22,8 @@ hdf5: - 1.10.6 jpeg: - '9' +libzip: +- '1' macos_machine: - x86_64-apple-darwin13.4.0 mpi: diff --git a/.ci_support/win_64_.yaml b/.ci_support/win_64_.yaml index 1420906a..952186ed 100644 --- a/.ci_support/win_64_.yaml +++ b/.ci_support/win_64_.yaml @@ -16,6 +16,8 @@ hdf5: - 1.10.6 jpeg: - '9' +libzip: +- '1' mpi: - nompi pin_run_as_build: From 539e0d992ce87ac129eabc4e90f2cc0de62c4e29 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Fri, 9 Apr 2021 15:12:58 -0300 Subject: [PATCH 07/15] empty skip --- recipe/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/recipe/build.sh b/recipe/build.sh index cfc2d7f0..d21d71a3 100644 --- a/recipe/build.sh +++ b/recipe/build.sh @@ -110,6 +110,7 @@ cmake -DCMAKE_INSTALL_PREFIX=${PREFIX} \ make install -j${CPU_COUNT} ${VERBOSE_CM} ctest -VV --output-on-failure -j${CPU_COUNT} +SKIP="" # Skip failing test on ppc74le if [ ${target_platform} == "linux-ppc64le" ]; then SKIP="-E nc_test" From b899096d80676e592786f9d2e8246e0f53bcba82 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Fri, 9 Apr 2021 14:53:35 -0500 Subject: [PATCH 08/15] cleanup --- recipe/build.sh | 76 +++++++++++++++++++++---------------------------- 1 file changed, 32 insertions(+), 44 deletions(-) diff --git a/recipe/build.sh b/recipe/build.sh index d21d71a3..03977e17 100644 --- a/recipe/build.sh +++ b/recipe/build.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -x + if [[ ! -z "$mpi" && "$mpi" != "nompi" ]]; then export PARALLEL="-DENABLE_PARALLEL4=ON -DENABLE_PARALLEL_TESTS=ON" export CC=mpicc @@ -15,30 +17,24 @@ else PARALLEL="" fi -if [[ ${c_compiler} != "toolchain_c" ]]; then - declare -a CMAKE_PLATFORM_FLAGS - if [[ ${HOST} =~ .*darwin.* ]]; then - CMAKE_PLATFORM_FLAGS+=(-DCMAKE_OSX_SYSROOT="${CONDA_BUILD_SYSROOT}") - # We have a problem with over-stripping of dylibs in the test programs: - # nm ${PREFIX}/lib/libdf.dylib | grep error_top - # 000000000006197c S _error_top - # Then, despite this being linked to explicitly when creating the test programs: - # ./hdf4_test_tst_chunk_hdf4 - # dyld: Symbol not found: _error_top - # Referenced from: ${PREFIX}/lib/libmfhdf.0.dylib - # Expected in: flat namespace - # in ${PREFIX}/lib/libmfhdf.0.dylib - # Abort trap: 56 - # Now clearly libmfhdf should autoload libdf but it does not and that is not going to change: - # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=556439 - # .. so we must remove our unused stripping instead :-( - # (it may be possible to arrange this symbol to be in the 'D'ata section instead of 'S' - # (symbol in a section other than those above according to man nm), instead though - # or to fix ld64 so that it checks for symbols being used in this section). - export LDFLAGS=$(echo "${LDFLAGS}" | sed "s/-Wl,-dead_strip_dylibs//g") - else - CMAKE_PLATFORM_FLAGS+=(-DCMAKE_TOOLCHAIN_FILE="${RECIPE_DIR}/cross-linux.cmake") - fi +if [[ ${HOST} =~ .*darwin.* ]]; then + # We have a problem with over-stripping of dylibs in the test programs: + # nm ${PREFIX}/lib/libdf.dylib | grep error_top + # 000000000006197c S _error_top + # Then, despite this being linked to explicitly when creating the test programs: + # ./hdf4_test_tst_chunk_hdf4 + # dyld: Symbol not found: _error_top + # Referenced from: ${PREFIX}/lib/libmfhdf.0.dylib + # Expected in: flat namespace + # in ${PREFIX}/lib/libmfhdf.0.dylib + # Abort trap: 56 + # Now clearly libmfhdf should autoload libdf but it does not and that is not going to change: + # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=556439 + # .. so we must remove our unused stripping instead :-( + # (it may be possible to arrange this symbol to be in the 'D'ata section instead of 'S' + # (symbol in a section other than those above according to man nm), instead though + # or to fix ld64 so that it checks for symbols being used in this section). + export LDFLAGS=$(echo "${LDFLAGS}" | sed "s/-Wl,-dead_strip_dylibs//g") fi # hmaarrfk, 2020/05/04: @@ -61,7 +57,7 @@ else fi # Build static. -cmake -DCMAKE_INSTALL_PREFIX=${PREFIX} \ +cmake ${CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${PREFIX} \ -DCMAKE_INSTALL_LIBDIR="lib" \ -DCMAKE_PREFIX_PATH=${PREFIX} \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ @@ -72,11 +68,8 @@ cmake -DCMAKE_INSTALL_PREFIX=${PREFIX} \ -DENABLE_TESTS=ON \ -DBUILD_UTILITIES=ON \ -DENABLE_DOXYGEN=OFF \ - -DCMAKE_C_FLAGS_RELEASE=${CFLAGS} \ - -DCMAKE_C_FLAGS_DEBUG=${CFLAGS} \ -DENABLE_CDF5=ON \ -DENABLE_BYTERANGE=ON \ - ${CMAKE_PLATFORM_FLAGS[@]} \ ${PARALLEL} \ -DENABLE_NCZARR=on \ -DENABLE_NCZARR_S3=off \ @@ -87,7 +80,7 @@ make install -j${CPU_COUNT} ${VERBOSE_CM} make clean # Build shared. -cmake -DCMAKE_INSTALL_PREFIX=${PREFIX} \ +cmake ${CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${PREFIX} \ -DCMAKE_INSTALL_LIBDIR="lib" \ -DCMAKE_PREFIX_PATH=${PREFIX} \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ @@ -98,10 +91,7 @@ cmake -DCMAKE_INSTALL_PREFIX=${PREFIX} \ -DENABLE_TESTS=ON \ -DBUILD_UTILITIES=ON \ -DENABLE_DOXYGEN=OFF \ - -DCMAKE_C_FLAGS_RELEASE=${CFLAGS} \ - -DCMAKE_C_FLAGS_DEBUG=${CFLAGS} \ -DENABLE_CDF5=ON \ - ${CMAKE_PLATFORM_FLAGS[@]} \ ${PARALLEL} \ -DENABLE_NCZARR=on \ -DENABLE_NCZARR_S3=off \ @@ -118,16 +108,14 @@ fi ctest -VV --output-on-failure -j${CPU_COUNT} "${SKIP}" -if [[ ${c_compiler} != "toolchain_c" ]]; then - # Fix build paths in cmake artifacts - for fname in `ls ${PREFIX}/lib/cmake/netCDF/*`; do - sed -i.bak "s#${CONDA_BUILD_SYSROOT}/usr/lib/lib\([a-z]*\).so#\1#g" ${fname} - sed -i.bak "s#/Applications/Xcode_.*app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.*sdk/usr/lib/lib\([a-z]*\).dylib#\1#g" ${fname} - rm ${fname}.bak - cat ${fname} - done +# Fix build paths in cmake artifacts +for fname in `ls ${PREFIX}/lib/cmake/netCDF/*`; do + sed -i.bak "s#${CONDA_BUILD_SYSROOT}/usr/lib/lib\([a-z]*\).so#\1#g" ${fname} + sed -i.bak "s#/Applications/Xcode_.*app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.*sdk/usr/lib/lib\([a-z]*\).dylib#\1#g" ${fname} + rm ${fname}.bak + cat ${fname} +done - # Fix build paths in nc-config - sed -i.bak "s#${BUILD_PREFIX}/bin/${CC}#${CC}#g" ${PREFIX}/bin/nc-config - rm ${PREFIX}/bin/nc-config.bak -fi +# Fix build paths in nc-config +sed -i.bak "s#${BUILD_PREFIX}/bin/${CC}#${CC}#g" ${PREFIX}/bin/nc-config +rm ${PREFIX}/bin/nc-config.bak From f9f9333da6be4fc3ba6cbafb7f066a502125ddf1 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Fri, 9 Apr 2021 18:03:04 -0500 Subject: [PATCH 09/15] remove extra quotes --- recipe/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipe/build.sh b/recipe/build.sh index 03977e17..4d1769fd 100644 --- a/recipe/build.sh +++ b/recipe/build.sh @@ -106,7 +106,7 @@ if [ ${target_platform} == "linux-ppc64le" ]; then SKIP="-E nc_test" fi -ctest -VV --output-on-failure -j${CPU_COUNT} "${SKIP}" +ctest -VV --output-on-failure -j${CPU_COUNT} ${SKIP} # Fix build paths in cmake artifacts for fname in `ls ${PREFIX}/lib/cmake/netCDF/*`; do From e93a38d8d112d135e323818067fb4eaf78064b02 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Fri, 9 Apr 2021 20:24:53 -0500 Subject: [PATCH 10/15] Update build.sh --- recipe/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipe/build.sh b/recipe/build.sh index 4d1769fd..1adc801a 100644 --- a/recipe/build.sh +++ b/recipe/build.sh @@ -102,7 +102,7 @@ ctest -VV --output-on-failure -j${CPU_COUNT} SKIP="" # Skip failing test on ppc74le -if [ ${target_platform} == "linux-ppc64le" ]; then +if [[ ${target_platform} == "linux-ppc64le" ]]; then SKIP="-E nc_test" fi From 5e73e721ff7e4939b2d0030c3064b39b27ea8b43 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Sat, 10 Apr 2021 17:02:26 -0300 Subject: [PATCH 11/15] remove extra test line --- recipe/build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/recipe/build.sh b/recipe/build.sh index 1adc801a..d7c9c006 100644 --- a/recipe/build.sh +++ b/recipe/build.sh @@ -98,7 +98,6 @@ cmake ${CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${PREFIX} \ -DENABLE_NCZARR_S3_TESTS=off \ ${SRC_DIR} make install -j${CPU_COUNT} ${VERBOSE_CM} -ctest -VV --output-on-failure -j${CPU_COUNT} SKIP="" # Skip failing test on ppc74le From bc18856d4b55048f27ba015226f621f11907a726 Mon Sep 17 00:00:00 2001 From: Ryan May Date: Mon, 12 Apr 2021 14:43:44 -0600 Subject: [PATCH 12/15] Eliminate compiler optimzation on ppc64le --- recipe/build.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/recipe/build.sh b/recipe/build.sh index d7c9c006..34d84922 100644 --- a/recipe/build.sh +++ b/recipe/build.sh @@ -50,6 +50,11 @@ else ENABLE_HDF4=ON fi +if [[ ${target_platform} == "linux-ppc64le" ]]; then + export CFLAGS=$(echo "${CFLAGS}" | sed "s/-O3/-O0/g") + echo CHANGING CFLAGS: ${CFLAGS} +fi + if [[ ${DEBUG_C} == yes ]]; then CMAKE_BUILD_TYPE=Debug else From f7a51c0168f5d6731c1f7d034c2bf0c2e4dcb1ef Mon Sep 17 00:00:00 2001 From: Ryan May Date: Mon, 12 Apr 2021 16:37:39 -0600 Subject: [PATCH 13/15] Try harder to replace -O3 --- recipe/build.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/recipe/build.sh b/recipe/build.sh index 34d84922..20367a88 100644 --- a/recipe/build.sh +++ b/recipe/build.sh @@ -51,8 +51,10 @@ else fi if [[ ${target_platform} == "linux-ppc64le" ]]; then - export CFLAGS=$(echo "${CFLAGS}" | sed "s/-O3/-O0/g") + export CFLAGS=${CFLAGS//-O3/-O0} echo CHANGING CFLAGS: ${CFLAGS} + export CMAKE_ARGS=${CMAKE_ARGS//-O3/-O0} + echo CHANGING CMAKE ARGS: ${CMAKE_ARGS} fi if [[ ${DEBUG_C} == yes ]]; then From c706885d29fc00d65c58f7377201ed5b7a46ac88 Mon Sep 17 00:00:00 2001 From: Ryan May Date: Mon, 12 Apr 2021 17:09:46 -0600 Subject: [PATCH 14/15] Keep CMake from adding -O3 on ppc64le --- recipe/build.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/recipe/build.sh b/recipe/build.sh index 20367a88..cf525587 100644 --- a/recipe/build.sh +++ b/recipe/build.sh @@ -50,19 +50,18 @@ else ENABLE_HDF4=ON fi -if [[ ${target_platform} == "linux-ppc64le" ]]; then - export CFLAGS=${CFLAGS//-O3/-O0} - echo CHANGING CFLAGS: ${CFLAGS} - export CMAKE_ARGS=${CMAKE_ARGS//-O3/-O0} - echo CHANGING CMAKE ARGS: ${CMAKE_ARGS} -fi - if [[ ${DEBUG_C} == yes ]]; then CMAKE_BUILD_TYPE=Debug else CMAKE_BUILD_TYPE=Release fi +if [[ ${target_platform} == "linux-ppc64le" ]]; then + export CFLAGS=${CFLAGS//-O3/-O0} + # This is the easiest way to get CMake to stop appending -O3 + CMAKE_BUILD_TYPE=None +fi + # Build static. cmake ${CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${PREFIX} \ -DCMAKE_INSTALL_LIBDIR="lib" \ From cde315d8aa888eaddca94dff1dea739da28ef347 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Fri, 16 Apr 2021 15:25:33 -0500 Subject: [PATCH 15/15] Remove skips --- recipe/build.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/recipe/build.sh b/recipe/build.sh index cf525587..a84ca94c 100644 --- a/recipe/build.sh +++ b/recipe/build.sh @@ -106,10 +106,6 @@ cmake ${CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${PREFIX} \ make install -j${CPU_COUNT} ${VERBOSE_CM} SKIP="" -# Skip failing test on ppc74le -if [[ ${target_platform} == "linux-ppc64le" ]]; then - SKIP="-E nc_test" -fi ctest -VV --output-on-failure -j${CPU_COUNT} ${SKIP}