From 9c05d4dba113f3b3c6a2cd19c724b655c26ed6c2 Mon Sep 17 00:00:00 2001 From: Vicente Adolfo Bolea Sanchez Date: Fri, 26 Jul 2024 18:07:22 -0400 Subject: [PATCH] ci,macos: use conda for all deps --- .github/workflows/everything.yml | 30 +++++---- .github/workflows/pypackaging.yml | 2 + ...i-macos-12-xcode13_4_1-shared-serial.cmake | 28 --------- ...i-macos-12-xcode13_4_1-static-serial.cmake | 52 ---------------- .../ci-macos-13-xcode14_2-shared-serial.cmake | 8 ++- scripts/ci/gh-actions/conda-env-macos.yml | 10 +++ scripts/ci/gh-actions/macos-setup.sh | 61 +++++++++---------- scripts/ci/setup-run/ci-macOS.sh | 5 ++ 8 files changed, 73 insertions(+), 123 deletions(-) delete mode 100644 scripts/ci/cmake/ci-macos-12-xcode13_4_1-shared-serial.cmake delete mode 100644 scripts/ci/cmake/ci-macos-12-xcode13_4_1-static-serial.cmake create mode 100644 scripts/ci/gh-actions/conda-env-macos.yml create mode 100644 scripts/ci/setup-run/ci-macOS.sh diff --git a/.github/workflows/everything.yml b/.github/workflows/everything.yml index 19424b04ae..bed42a40e8 100644 --- a/.github/workflows/everything.yml +++ b/.github/workflows/everything.yml @@ -110,7 +110,7 @@ jobs: ubuntu: needs: [format, git_checks] - if: needs.git_checks.outputs.num_code_changes > 0 + if: false runs-on: ubuntu-20.04 container: @@ -208,7 +208,7 @@ jobs: el8: needs: [format, git_checks] - if: needs.git_checks.outputs.num_code_changes > 0 + if: false runs-on: ubuntu-latest container: @@ -276,15 +276,18 @@ jobs: runs-on: ${{ matrix.image }} env: - GH_YML_JOBNAME: ${{ matrix.image }}-${{ matrix.compiler }}-${{ matrix.shared }}-${{ matrix.parallel }} + # Only way to source a file in a non interactive/non login shell. + BASH_ENV: "/Users/runner/.bash_profile" + CCACHE_BASEDIR: "${GITHUB_WORKSPACE}" + CCACHE_COMPRESSLEVEL: 6 + CCACHE_COMPRESS: true + CCACHE_DIR: "${GITHUB_WORKSPACE}/.ccache" GH_YML_BASE_OS: macOS - GH_YML_MATRIX_OS: ${{ matrix.image }} + GH_YML_JOBNAME: ${{ matrix.image }}-${{ matrix.compiler }}-${{ matrix.shared }}-${{ matrix.parallel }} + GH_YML_MATRIX_ARCH : ${{ matrix.arch }} GH_YML_MATRIX_COMPILER: ${{ matrix.compiler }} + GH_YML_MATRIX_OS: ${{ matrix.image }} GH_YML_MATRIX_PARALLEL: ${{ matrix.parallel }} - CCACHE_BASEDIR: "${GITHUB_WORKSPACE}" - CCACHE_DIR: "${GITHUB_WORKSPACE}/.ccache" - CCACHE_COMPRESS: true - CCACHE_COMPRESSLEVEL: 6 strategy: fail-fast: false @@ -292,6 +295,10 @@ jobs: image: [macos-13, macos-14] shared: [static, shared] include: + - image: macos-13 + arch: x64 + - image: macos-14 + arch: arm64 - parallel: serial - image: macos-13 compiler: xcode14_2 @@ -341,7 +348,7 @@ jobs: windows: needs: [format, git_checks] - if: needs.git_checks.outputs.num_code_changes > 0 + if: false runs-on: ${{ matrix.image }} env: @@ -400,7 +407,7 @@ jobs: docker: needs: [format, git_checks] - if: needs.git_checks.outputs.num_code_changes > 0 + if: false runs-on: ubuntu-latest strategy: @@ -465,7 +472,7 @@ jobs: # standard container job. contract: needs: [format, git_checks, docker] - if: needs.git_checks.outputs.num_code_changes > 0 + if: false runs-on: ubuntu-latest strategy: @@ -521,6 +528,7 @@ jobs: analyze: needs: [format, git_checks] + if: false name: CodeQL runs-on: ubuntu-latest container: diff --git a/.github/workflows/pypackaging.yml b/.github/workflows/pypackaging.yml index 022807108e..f7d7f75c00 100644 --- a/.github/workflows/pypackaging.yml +++ b/.github/workflows/pypackaging.yml @@ -22,6 +22,7 @@ env: jobs: make_sdist: name: Make SDist + if: false runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -41,6 +42,7 @@ jobs: build_wheels: name: Wheel on ${{ matrix.os }} runs-on: ${{ matrix.os }} + if: false strategy: fail-fast: false matrix: diff --git a/scripts/ci/cmake/ci-macos-12-xcode13_4_1-shared-serial.cmake b/scripts/ci/cmake/ci-macos-12-xcode13_4_1-shared-serial.cmake deleted file mode 100644 index 536846b256..0000000000 --- a/scripts/ci/cmake/ci-macos-12-xcode13_4_1-shared-serial.cmake +++ /dev/null @@ -1,28 +0,0 @@ -# Client maintainer: vicente.bolea@kitware.com -set(ENV{CC} clang) -set(ENV{CXX} clang++) -set(ENV{FC} gfortran-11) - -set(dashboard_cache " -BUILD_TESTING:BOOL=ON -ADIOS2_BUILD_EXAMPLES:BOOL=ON - -ADIOS2_USE_Fortran:BOOL=ON -ADIOS2_USE_MPI:BOOL=OFF -ADISO2_USE_Python:BOOL=ON - -CMAKE_C_COMPILER_LAUNCHER=ccache -CMAKE_CXX_COMPILER_LAUNCHER=ccache -CMAKE_C_FLAGS:STRING=-Wall -CMAKE_CXX_FLAGS:STRING=-Wall -CMAKE_Fortran_FLAGS:STRING=-Wall -") - -set(ENV{MACOSX_DEPLOYMENT_TARGET} "12.1") -set(CTEST_CMAKE_GENERATOR "Ninja") -list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") -set(CTEST_TEST_ARGS - # Unclear why these tests currently die. Disabling until it can be addressed. - EXCLUDE "Install.Make.*" -) -include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/cmake/ci-macos-12-xcode13_4_1-static-serial.cmake b/scripts/ci/cmake/ci-macos-12-xcode13_4_1-static-serial.cmake deleted file mode 100644 index 68e43b31b3..0000000000 --- a/scripts/ci/cmake/ci-macos-12-xcode13_4_1-static-serial.cmake +++ /dev/null @@ -1,52 +0,0 @@ -# Client maintainer: vicente.bolea@kitware.com -set(ENV{CC} clang) -set(ENV{CXX} clang++) -set(ENV{FC} gfortran-11) - -set(dashboard_cache " -BUILD_SHARED_LIBS=OFF -CMAKE_BUILD_TYPE:STRING=Release -BUILD_TESTING:BOOL=ON -ADIOS2_BUILD_EXAMPLES:BOOL=OFF - -ADIOS2_USE_AWSSDK:STRING=OFF -ADIOS2_USE_Blosc2:STRING=OFF -ADIOS2_USE_Blosc2:BOOL=OFF -ADIOS2_USE_BZip2:BOOL=OFF -ADIOS2_USE_Catalyst:STRING=OFF -ADIOS2_USE_CUDA:STRING=OFF -ADIOS2_USE_DAOS:STRING=OFF -ADIOS2_USE_DataMan:BOOL=OFF -ADIOS2_USE_DataSpaces:STRING=OFF -ADIOS2_USE_Fortran:BOOL=OFF -ADIOS2_USE_HDF5:BOOL=OFF -ADIOS2_USE_HDF5_VOL:STRING=OFF -ADIOS2_USE_IME:STRING=OFF -ADIOS2_USE_Kokkos:STRING=OFF -ADIOS2_USE_LIBPRESSIO:STRING=OFF -ADIOS2_USE_MGARD:STRING=OFF -ADIOS2_USE_MHS:STRING=OFF -ADIOS2_USE_MPI:BOOL=OFF -ADIOS2_USE_PNG:BOOL=OFF -ADIOS2_USE_Python:BOOL=OFF -ADIOS2_USE_SSC:BOOL=OFF -ADIOS2_USE_SST:BOOL=OFF -ADIOS2_USE_SZ:BOOL=OFF -ADIOS2_USE_ZeroMQ:STRING=OFF -ADIOS2_USE_ZFP:BOOL=OFF - -CMAKE_C_COMPILER_LAUNCHER=ccache -CMAKE_CXX_COMPILER_LAUNCHER=ccache -CMAKE_C_FLAGS:STRING=-Wall -CMAKE_CXX_FLAGS:STRING=-Wall -CMAKE_Fortran_FLAGS:STRING=-Wall -") - -set(ENV{MACOSX_DEPLOYMENT_TARGET} "12.1") -set(CTEST_CMAKE_GENERATOR "Ninja") -list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") -set(CTEST_TEST_ARGS - # Unclear why these tests currently die. Disabling until it can be addressed. - EXCLUDE "Install.Make.*" -) -include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/cmake/ci-macos-13-xcode14_2-shared-serial.cmake b/scripts/ci/cmake/ci-macos-13-xcode14_2-shared-serial.cmake index e2496e6ff9..cc69488adb 100644 --- a/scripts/ci/cmake/ci-macos-13-xcode14_2-shared-serial.cmake +++ b/scripts/ci/cmake/ci-macos-13-xcode14_2-shared-serial.cmake @@ -9,7 +9,13 @@ ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON ADIOS2_USE_MPI:BOOL=OFF -ADISO2_USE_Python:BOOL=ON +ADIOS2_USE_Blosc2:BOOL=ON +ADIOS2_USE_Bzip2:BOOL=ON +ADIOS2_USE_DataMan:BOOL=ON +ADIOS2_USE_HDF5:BOOL=ON +ADIOS2_USE_Python:BOOL=ON +ADIOS2_USE_Sodium:BOOL=ON +ADIOS2_USE_ZeroMQ:BOOL=ON CMAKE_C_COMPILER_LAUNCHER=ccache CMAKE_CXX_COMPILER_LAUNCHER=ccache diff --git a/scripts/ci/gh-actions/conda-env-macos.yml b/scripts/ci/gh-actions/conda-env-macos.yml new file mode 100644 index 0000000000..96fc42ce10 --- /dev/null +++ b/scripts/ci/gh-actions/conda-env-macos.yml @@ -0,0 +1,10 @@ +name: adios2 +variables: + CMAKE_PREFIX_PATH: "/Users/runner/miniconda3/envs/adios2/lib:$CMAKE_PREFIX_PATH" +dependencies: + - c-blosc2 + - ccache + - hdf5 + - ninja + - numpy>=1.19 + - zeromq diff --git a/scripts/ci/gh-actions/macos-setup.sh b/scripts/ci/gh-actions/macos-setup.sh index 0eb6850f20..40e9bd61c9 100755 --- a/scripts/ci/gh-actions/macos-setup.sh +++ b/scripts/ci/gh-actions/macos-setup.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e +set -xe echo "Setting up default XCode version" if [ -z "${GH_YML_MATRIX_COMPILER}" ] @@ -15,40 +15,39 @@ then exit 2 fi sudo xcode-select --switch "/Applications/Xcode_${XCODE_VER}.app" +sudo ln -v -s "$(which gfortran-11)" /usr/local/bin/gfortran -echo "Installing CMake" - -{ - readonly version="3.24.2" - readonly checksum="efb11a78c064dd7c54a50b8da247254d252112c402c6e48cb7db3f9c84a4e5ad" - readonly pkg="cmake-${version}-macos-universal.tar.gz" - echo "${checksum} ${pkg}" > cmake.sha256sum - - curl -OL "https://github.com/Kitware/CMake/releases/download/v${version}/${pkg}" - shasum -a 256 --check cmake.sha256sum - - sudo tar -xvzf ${pkg} --strip-components 1 -C /Applications/ -} - +echo "Installing Miniconda" -echo "Installing Ninja" -brew install ninja +if [ "${RUNNER_ARCH}" = "X64" ] +then + readonly checksum="6d7c1cc138adfc4bb2ccbb8a22eb8e9eb13a366b6af0d63245b643e6c3a3c708" + readonly pkg="Miniconda3-py310_24.5.0-0-MacOSX-x86_64.sh" +elif [ "${RUNNER_ARCH}" = "ARM64" ] +then + readonly checksum="e422602aa19140c600b5889e5b41a0d7187640107ea82fcb5da857dd25330148" + readonly pkg="Miniconda3-py310_24.5.0-0-MacOSX-arm64.sh" +else + echo "Error: unknown platform: ${RUNNER_ARCH} " + exit 3 +fi +echo "${checksum} ${pkg}" > miniconda.sha256sum -echo "Installing GCC" -brew install gcc -sudo ln -v -s "$(which gfortran-11)" /usr/local/bin/gfortran +curl -OL "https://repo.anaconda.com/miniconda/${pkg}" +shasum -a 256 --check miniconda.sha256sum +bash "./${pkg}" -b -echo "Installing blosc compression" -brew install c-blosc +# shellcheck source=/dev/null +source "/Users/runner/miniconda3/bin/activate" -echo "Installing python3" -brew install python numpy +# Canonical installation of Miniconda +conda init --all +conda update --all -y +conda config --add channels conda-forge +conda config --set channel_priority strict -echo "Installing ccache" -brew install ccache +conda env create -f "gha/scripts/ci/gh-actions/conda-env-macos.yml" -if [[ "$GH_YML_JOBNAME" =~ -ompi ]] -then - echo "Installing OpenMPI" - brew install openmpi mpi4py -fi +conda list +conda info --verbose +echo 'conda activate adios2' >> /Users/runner/.bash_profile diff --git a/scripts/ci/setup-run/ci-macOS.sh b/scripts/ci/setup-run/ci-macOS.sh new file mode 100644 index 0000000000..098162be73 --- /dev/null +++ b/scripts/ci/setup-run/ci-macOS.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set +u +# We do not use conda cmake, thus we have to hint cmake where is +# conda root dir. +conda activate adios2