From f9fb9a8c02c76fe8ca8036f286b0fdee5d8cd5f4 Mon Sep 17 00:00:00 2001 From: John Muchovej <5000729+jmuchovej@users.noreply.github.com> Date: Wed, 9 Jun 2021 21:04:23 -0400 Subject: [PATCH 1/6] Upgrade from Julia 1.1.1 to 1.6.1 --- recipe/meta.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 161442db..b5bf9113 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -1,6 +1,6 @@ {% set name = "julia" %} -{% set version = "1.1.1" %} -{% set sha256 = "0155a33448cb2334ae7487a35a2416a4205da4541b90fc9f97208da11e273c9e" %} +{% set version = "1.6.1" %} +{% set sha256 = "2efc3f3485b837d2d77ed684ef883c55c66cb02edbaab7ffd120375cb86317da" %} package: name: {{ name|lower }} @@ -13,7 +13,7 @@ source: build: skip: True # [win] - number: 6 + number: 0 features: requirements: From e59adf19f0d4b85f85d4557a1c926ad3220fbddc Mon Sep 17 00:00:00 2001 From: conda-forge-linter Date: Thu, 10 Jun 2021 01:06:32 +0000 Subject: [PATCH 2/6] MNT: Re-rendered with conda-build 3.21.4, conda-smithy 3.10.1, and conda-forge-pinning 2021.06.09.12.51.26 --- .ci_support/linux_64_.yaml | 2 +- .ci_support/migrations/pcre2_1036.yaml | 12 ------ .ci_support/osx_64_.yaml | 2 +- .scripts/build_steps.sh | 27 +++++++------ .scripts/logging_utils.sh | 9 ++++- .scripts/run_docker_build.sh | 19 ++++++---- .scripts/run_osx_build.sh | 52 ++++++++++++++++---------- README.md | 1 + build-locally.py | 27 +++++++++++-- 9 files changed, 95 insertions(+), 56 deletions(-) delete mode 100644 .ci_support/migrations/pcre2_1036.yaml diff --git a/.ci_support/linux_64_.yaml b/.ci_support/linux_64_.yaml index 3a2a5ab8..770460fe 100644 --- a/.ci_support/linux_64_.yaml +++ b/.ci_support/linux_64_.yaml @@ -52,7 +52,7 @@ pin_run_as_build: zlib: max_pin: x.x suitesparse: -- '5.6' +- '5' target_platform: - linux-64 zip_keys: diff --git a/.ci_support/migrations/pcre2_1036.yaml b/.ci_support/migrations/pcre2_1036.yaml deleted file mode 100644 index 428b241d..00000000 --- a/.ci_support/migrations/pcre2_1036.yaml +++ /dev/null @@ -1,12 +0,0 @@ -migrator_ts: 1614603878 -__migrator: - kind: - version - migration_number: - 1 - bump_number: - 1 - -pcre2: - - '10.36' - diff --git a/.ci_support/osx_64_.yaml b/.ci_support/osx_64_.yaml index b00e822f..c2447382 100644 --- a/.ci_support/osx_64_.yaml +++ b/.ci_support/osx_64_.yaml @@ -52,7 +52,7 @@ pin_run_as_build: zlib: max_pin: x.x suitesparse: -- '5.6' +- '5' target_platform: - osx-64 zip_keys: diff --git a/.scripts/build_steps.sh b/.scripts/build_steps.sh index 522eabcf..71a06860 100755 --- a/.scripts/build_steps.sh +++ b/.scripts/build_steps.sh @@ -10,9 +10,10 @@ export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/feedstock_root}" source ${FEEDSTOCK_ROOT}/.scripts/logging_utils.sh -endgroup "Start Docker" +( endgroup "Start Docker" ) 2> /dev/null + +( startgroup "Configuring conda" ) 2> /dev/null -startgroup "Configuring conda" export PYTHONUNBUFFERED=1 export RECIPE_ROOT="${RECIPE_ROOT:-/home/conda/recipe_root}" export CI_SUPPORT="${FEEDSTOCK_ROOT}/.ci_support" @@ -36,34 +37,38 @@ source run_conda_forge_build_setup # make the build number clobber make_build_number "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" -endgroup "Configuring conda" + +( endgroup "Configuring conda" ) 2> /dev/null if [[ "${BUILD_WITH_CONDA_DEBUG:-0}" == 1 ]]; then - startgroup "Running conda debug" if [[ "x${BUILD_OUTPUT_ID:-}" != "x" ]]; then EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --output-id ${BUILD_OUTPUT_ID}" fi conda debug "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \ ${EXTRA_CB_OPTIONS:-} \ --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" - endgroup "Running conda debug" + # Drop into an interactive shell /bin/bash else - startgroup "Running conda $BUILD_CMD" conda $BUILD_CMD "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \ --suppress-variables ${EXTRA_CB_OPTIONS:-} \ --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" - endgroup "Running conda build" - startgroup "Validating outputs" + ( startgroup "Validating outputs" ) 2> /dev/null + validate_recipe_outputs "${FEEDSTOCK_NAME}" - endgroup "Validating outputs" + + ( endgroup "Validating outputs" ) 2> /dev/null + + ( startgroup "Uploading packages" ) 2> /dev/null if [[ "${UPLOAD_PACKAGES}" != "False" ]]; then - startgroup "Uploading packages" upload_package --validate --feedstock-name="${FEEDSTOCK_NAME}" "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" - endgroup "Uploading packages" fi + + ( endgroup "Uploading packages" ) 2> /dev/null fi +( startgroup "Final checks" ) 2> /dev/null + touch "${FEEDSTOCK_ROOT}/build_artifacts/conda-forge-build-done-${CONFIG}" \ No newline at end of file diff --git a/.scripts/logging_utils.sh b/.scripts/logging_utils.sh index a53ef3f2..57bc95c2 100644 --- a/.scripts/logging_utils.sh +++ b/.scripts/logging_utils.sh @@ -13,18 +13,23 @@ function startgroup { travis ) echo "$1" echo -en 'travis_fold:start:'"${1// /}"'\\r';; + github_actions ) + echo "::group::$1";; * ) echo "$1";; esac -} +} 2> /dev/null function endgroup { # End a foldable group of log lines # Pass a single argument, quoted + case ${CI:-} in azure ) echo "##[endgroup]";; travis ) echo -en 'travis_fold:end:'"${1// /}"'\\r';; + github_actions ) + echo "::endgroup::";; esac -} +} 2> /dev/null diff --git a/.scripts/run_docker_build.sh b/.scripts/run_docker_build.sh index 16d6c15a..244ccd1f 100755 --- a/.scripts/run_docker_build.sh +++ b/.scripts/run_docker_build.sh @@ -7,7 +7,7 @@ source .scripts/logging_utils.sh -startgroup "Configure Docker" +( startgroup "Configure Docker" ) 2> /dev/null set -xeo pipefail @@ -52,11 +52,11 @@ if [ -z "${DOCKER_IMAGE}" ]; then echo "WARNING: DOCKER_IMAGE variable not set and shyaml not installed. Trying to parse with coreutils" DOCKER_IMAGE=$(cat .ci_support/${CONFIG}.yaml | grep '^docker_image:$' -A 1 | tail -n 1 | cut -b 3-) if [ "${DOCKER_IMAGE}" = "" ]; then - echo "No docker_image entry found in ${CONFIG}. Falling back to condaforge/linux-anvil-comp7" - DOCKER_IMAGE="condaforge/linux-anvil-comp7" + echo "No docker_image entry found in ${CONFIG}. Falling back to quay.io/condaforge/linux-anvil-comp7" + DOCKER_IMAGE="quay.io/condaforge/linux-anvil-comp7" fi else - DOCKER_IMAGE="$(cat "${FEEDSTOCK_ROOT}/.ci_support/${CONFIG}.yaml" | shyaml get-value docker_image.0 condaforge/linux-anvil-comp7 )" + DOCKER_IMAGE="$(cat "${FEEDSTOCK_ROOT}/.ci_support/${CONFIG}.yaml" | shyaml get-value docker_image.0 quay.io/condaforge/linux-anvil-comp7 )" fi fi @@ -69,9 +69,11 @@ DOCKER_RUN_ARGS="${CONDA_FORGE_DOCKER_RUN_ARGS}" if [ -z "${CI}" ]; then DOCKER_RUN_ARGS="-it ${DOCKER_RUN_ARGS}" fi -endgroup "Configure Docker" -startgroup "Start Docker" +( endgroup "Configure Docker" ) 2> /dev/null + +( startgroup "Start Docker" ) 2> /dev/null + export UPLOAD_PACKAGES="${UPLOAD_PACKAGES:-True}" docker run ${DOCKER_RUN_ARGS} \ -v "${RECIPE_ROOT}":/home/conda/recipe_root:rw,z,delegated \ @@ -94,4 +96,7 @@ docker run ${DOCKER_RUN_ARGS} \ /home/conda/feedstock_root/${PROVIDER_DIR}/build_steps.sh # verify that the end of the script was reached -test -f "$DONE_CANARY" \ No newline at end of file +test -f "$DONE_CANARY" + +# This closes the last group opened in `build_steps.sh` +( endgroup "Final checks" ) 2> /dev/null \ No newline at end of file diff --git a/.scripts/run_osx_build.sh b/.scripts/run_osx_build.sh index 614ca844..96d02787 100755 --- a/.scripts/run_osx_build.sh +++ b/.scripts/run_osx_build.sh @@ -2,19 +2,24 @@ source .scripts/logging_utils.sh -set -x +set -xe + +MINIFORGE_HOME=${MINIFORGE_HOME:-${HOME}/miniforge3} + +( startgroup "Installing a fresh version of Miniforge" ) 2> /dev/null -startgroup "Installing a fresh version of Miniforge" MINIFORGE_URL="https://github.com/conda-forge/miniforge/releases/latest/download" MINIFORGE_FILE="Miniforge3-MacOSX-x86_64.sh" curl -L -O "${MINIFORGE_URL}/${MINIFORGE_FILE}" -bash $MINIFORGE_FILE -b -endgroup "Installing a fresh version of Miniforge" +bash $MINIFORGE_FILE -b -p ${MINIFORGE_HOME} + +( endgroup "Installing a fresh version of Miniforge" ) 2> /dev/null + +( startgroup "Configuring conda" ) 2> /dev/null -startgroup "Configuring conda" BUILD_CMD=build -source ${HOME}/miniforge3/etc/profile.d/conda.sh +source ${MINIFORGE_HOME}/etc/profile.d/conda.sh conda activate base echo -e "\n\nInstalling conda-forge-ci-setup=3 and conda-build." @@ -24,32 +29,41 @@ conda install -n base --quiet --yes "conda-forge-ci-setup=3" conda-build pip ${G echo -e "\n\nSetting up the condarc and mangling the compiler." setup_conda_rc ./ ./recipe ./.ci_support/${CONFIG}.yaml -mangle_compiler ./ ./recipe .ci_support/${CONFIG}.yaml -echo -e "\n\nMangling homebrew in the CI to avoid conflicts." -/usr/bin/sudo mangle_homebrew -/usr/bin/sudo -k +if [[ "${CI:-}" != "" ]]; then + mangle_compiler ./ ./recipe .ci_support/${CONFIG}.yaml +fi + +if [[ "${CI:-}" != "" ]]; then + echo -e "\n\nMangling homebrew in the CI to avoid conflicts." + /usr/bin/sudo mangle_homebrew + /usr/bin/sudo -k +else + echo -e "\n\nNot mangling homebrew as we are not running in CI" +fi echo -e "\n\nRunning the build setup script." source run_conda_forge_build_setup -endgroup "Configuring conda" -set -e +( endgroup "Configuring conda" ) 2> /dev/null + -startgroup "Running conda $BUILD_CMD" echo -e "\n\nMaking the build clobber file" make_build_number ./ ./recipe ./.ci_support/${CONFIG}.yaml conda $BUILD_CMD ./recipe -m ./.ci_support/${CONFIG}.yaml --suppress-variables --clobber-file ./.ci_support/clobber_${CONFIG}.yaml ${EXTRA_CB_OPTIONS:-} -endgroup "Running conda build" -startgroup "Validating outputs" +( startgroup "Validating outputs" ) 2> /dev/null + validate_recipe_outputs "${FEEDSTOCK_NAME}" -endgroup "Validating outputs" + +( endgroup "Validating outputs" ) 2> /dev/null + +( startgroup "Uploading packages" ) 2> /dev/null if [[ "${UPLOAD_PACKAGES}" != "False" ]]; then - startgroup "Uploading packages" upload_package --validate --feedstock-name="${FEEDSTOCK_NAME}" ./ ./recipe ./.ci_support/${CONFIG}.yaml - endgroup "Uploading packages" -fi \ No newline at end of file +fi + +( endgroup "Uploading packages" ) 2> /dev/null \ No newline at end of file diff --git a/README.md b/README.md index 4efbc18b..2d633a3b 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ Installing `julia` from the `conda-forge` channel can be achieved by adding `con ``` conda config --add channels conda-forge +conda config --set channel_priority strict ``` Once the `conda-forge` channel has been enabled, `julia` can be installed with: diff --git a/build-locally.py b/build-locally.py index 3453cfe6..00eeb34b 100755 --- a/build-locally.py +++ b/build-locally.py @@ -7,6 +7,7 @@ import glob import subprocess from argparse import ArgumentParser +import platform def setup_environment(ns): @@ -16,6 +17,10 @@ def setup_environment(ns): os.environ["BUILD_WITH_CONDA_DEBUG"] = "1" if ns.output_id: os.environ["BUILD_OUTPUT_ID"] = ns.output_id + if "MINIFORGE_HOME" not in os.environ: + os.environ["MINIFORGE_HOME"] = os.path.join( + os.path.dirname(__file__), "miniforge3" + ) def run_docker_build(ns): @@ -23,6 +28,11 @@ def run_docker_build(ns): subprocess.check_call([script]) +def run_osx_build(ns): + script = ".scripts/run_osx_build.sh" + subprocess.check_call([script]) + + def verify_config(ns): valid_configs = { os.path.basename(f)[:-5] for f in glob.glob(".ci_support/*.yaml") @@ -46,10 +56,16 @@ def verify_config(ns): else: raise ValueError("config " + ns.config + " is not valid") # Remove the following, as implemented - if not ns.config.startswith("linux"): + if ns.config.startswith("win"): raise ValueError( - f"only Linux configs currently supported, got {ns.config}" + f"only Linux/macOS configs currently supported, got {ns.config}" ) + elif ns.config.startswith("osx") and platform.system() == "Darwin": + if "OSX_SDK_DIR" not in os.environ: + raise RuntimeError( + "Need OSX_SDK_DIR env variable set. Run 'export OSX_SDK_DIR=/opt'" + "to download the SDK automatically to '/opt/MacOSX.sdk'" + ) def main(args=None): @@ -68,7 +84,12 @@ def main(args=None): verify_config(ns) setup_environment(ns) - run_docker_build(ns) + if ns.config.startswith("linux") or ( + ns.config.startswith("osx") and platform.system() == "Linux" + ): + run_docker_build(ns) + elif ns.config.startswith("osx"): + run_osx_build(ns) if __name__ == "__main__": From c11c2f2e3507f5bb817b826585edd5025949026a Mon Sep 17 00:00:00 2001 From: John Muchovej <5000729+jmuchovej@users.noreply.github.com> Date: Wed, 9 Jun 2021 22:17:02 -0400 Subject: [PATCH 3/6] Attempting hotfix of make cleanall rec'd by azure --- recipe/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/recipe/build.sh b/recipe/build.sh index e83f8379..dc6f2a45 100644 --- a/recipe/build.sh +++ b/recipe/build.sh @@ -28,6 +28,7 @@ then export EXTRA_MAKEFLAGS="USE_SYSTEM_LIBUNWIND=0" fi +make cleanall # attempted resolution of build error on PCRE from Azure make -j 4 prefix=${PREFIX} MARCH=core2 sysconfigdir=${PREFIX}/etc NO_GIT=1 \ LIBBLAS=-lblas LIBBLASNAME=libblas LIBLAPACK=-llapack LIBLAPACKNAME=liblapack \ USE_SYSTEM_ARPACK=1 \ From ffeb708cf2ad48439e850ca8b6572eb95f8a186d Mon Sep 17 00:00:00 2001 From: John Muchovej <5000729+jmuchovej@users.noreply.github.com> Date: Wed, 9 Jun 2021 22:38:21 -0400 Subject: [PATCH 4/6] Upgrade PCRE2 to 10.36 Julia uses this in `deps/Versions.make` --- recipe/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index b5bf9113..874af292 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -40,7 +40,7 @@ requirements: - libosxunwind # [osx] - arpack - suitesparse - - pcre2 10.23 + - pcre2 10.36 run: - zlib @@ -54,7 +54,7 @@ requirements: - arpack - suitesparse - curl - - pcre2 10.23 + - pcre2 10.36 test: commands: From a67508e7300e3ad74b10d5cad7c8932c134849c1 Mon Sep 17 00:00:00 2001 From: John Muchovej <5000729+jmuchovej@users.noreply.github.com> Date: Mon, 14 Jun 2021 18:55:52 -0400 Subject: [PATCH 5/6] Attempting to force usage of system libnghttp2 Per https://github.com/JuliaLang/julia/pull/37618/files#diff-2592b7284deb7b3ee2572fbb4f110a2d868137ba3970d01a7ea3c7d07b808642, Julia 1.6 supports the use of the system `libnghttp2` while `make`-ing. --- recipe/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/recipe/build.sh b/recipe/build.sh index dc6f2a45..6e4ccc3b 100644 --- a/recipe/build.sh +++ b/recipe/build.sh @@ -46,6 +46,7 @@ make -j 4 prefix=${PREFIX} MARCH=core2 sysconfigdir=${PREFIX}/etc NO_GIT=1 \ USE_SYSTEM_PATCHELF=1 \ USE_SYSTEM_PCRE=1 \ USE_SYSTEM_SUITESPARSE=1 \ + USE_SYSTEM_NGHTTP2=1 \ ${EXTRA_MAKEFLAGS} \ TAGGED_RELEASE_BANNER="conda-forge-julia release" \ CC=$CC CXX=$CXX FC=$FC \ From 72c7c39bad273d910e86f06926c0c7caa2fd658b Mon Sep 17 00:00:00 2001 From: John Muchovej <5000729+jmuchovej@users.noreply.github.com> Date: Mon, 14 Jun 2021 19:11:10 -0400 Subject: [PATCH 6/6] Remove make `cleanall` and `USE_SYSTEM_HTTP2` --- recipe/build.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/recipe/build.sh b/recipe/build.sh index 6e4ccc3b..e83f8379 100644 --- a/recipe/build.sh +++ b/recipe/build.sh @@ -28,7 +28,6 @@ then export EXTRA_MAKEFLAGS="USE_SYSTEM_LIBUNWIND=0" fi -make cleanall # attempted resolution of build error on PCRE from Azure make -j 4 prefix=${PREFIX} MARCH=core2 sysconfigdir=${PREFIX}/etc NO_GIT=1 \ LIBBLAS=-lblas LIBBLASNAME=libblas LIBLAPACK=-llapack LIBLAPACKNAME=liblapack \ USE_SYSTEM_ARPACK=1 \ @@ -46,7 +45,6 @@ make -j 4 prefix=${PREFIX} MARCH=core2 sysconfigdir=${PREFIX}/etc NO_GIT=1 \ USE_SYSTEM_PATCHELF=1 \ USE_SYSTEM_PCRE=1 \ USE_SYSTEM_SUITESPARSE=1 \ - USE_SYSTEM_NGHTTP2=1 \ ${EXTRA_MAKEFLAGS} \ TAGGED_RELEASE_BANNER="conda-forge-julia release" \ CC=$CC CXX=$CXX FC=$FC \