diff --git a/.github/actions/setup-spack-stack/action.yaml b/.github/actions/setup-spack-stack/action.yaml index 7b3ac7058..de0b2749e 100644 --- a/.github/actions/setup-spack-stack/action.yaml +++ b/.github/actions/setup-spack-stack/action.yaml @@ -34,12 +34,18 @@ runs: shell: bash run: | if [[ "$RUNNER_OS" == "Linux" ]]; then + # Needed for the following apt-get install calls to work + sudo apt-get update + # Install Curl headers. Executable exists by default in spack external find. sudo apt-get install libcurl4-openssl-dev # Install git-lfs to avoid compilation errors of "go" with Intel sudo apt-get install git-lfs + # Install krb5 to avoid compilation errors of "hdf" on Ubuntu + sudo apt-get install krb5-user libkrb5-dev + if [[ "${{ inputs.compiler }}" == "intel"* ]]; then cd /tmp wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB @@ -49,6 +55,15 @@ runs: sudo apt-get update sudo apt-get install intel-oneapi-dev-utilities intel-oneapi-openmp intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic intel-oneapi-mpi-devel echo "source /opt/intel/oneapi/setvars.sh" >> ~/.bash_profile + # + # DH* + # For now, install qt5 only for Intel to shorten the build time + # without having to rerun all other CI tests. Later, should consider + # using https://github.com/jurplel/install-qt-action for all runners + # and get rid of the macOS homebrew installation/package config + #sudo apt-get install qt5-default qttools5-dev-tools + sudo apt-get install qt5-default + # *DH fi elif [[ "$RUNNER_OS" == "macOS" ]]; then # These are already installed @@ -107,14 +122,34 @@ runs: tar -xzf openmpi-${OPENMPI_VERSION}.tar.gz cd openmpi-${OPENMPI_VERSION} # --with-hwloc=internal --with-libevent=internal : https://www.open-mpi.org/faq/?category=building#libevent-or-hwloc-errors-when-linking-fortran - ./configure --prefix=$HOME/mpi --enable-mpi-fortran --enable-mpi-cxx --with-hwloc=internal --with-libevent=internal + if [[ "$RUNNER_OS" == "macOS" ]]; then + ./configure --prefix=$HOME/mpi \ + --enable-mpi-fortran --enable-mpi-cxx \ + --with-hwloc=internal --with-libevent=internal \ + --with-wrapper-ldflags="-Wl,-commons,use_dylibs" \ + LIBS="-Wl,-commons,use_dylibs" + else + ./configure --prefix=$HOME/mpi \ + --enable-mpi-fortran --enable-mpi-cxx \ + --with-hwloc=internal --with-libevent=internal + fi make -j2 make install elif [[ "${{ inputs.mpi }}" == "mpich"* ]]; then wget http://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz tar -xzf mpich-${MPICH_VERSION}.tar.gz cd mpich-${MPICH_VERSION} - ./configure --prefix=$HOME/mpi --enable-fortran --enable-cxx --with-device=ch4:ofi + if [[ "$RUNNER_OS" == "macOS" ]]; then + ./configure --prefix=$HOME/mpi \ + --enable-fortran --enable-cxx \ + --with-device=ch4:ofi \ + --enable-two-level-namespace \ + LIBS="-Wl,-commons,use_dylibs" + else + ./configure --prefix=$HOME/mpi \ + --enable-fortran --enable-cxx \ + --with-device=ch4:ofi + fi make -j2 make install fi @@ -128,7 +163,7 @@ runs: fi source setup.sh - ./create-env.py --site default --app ${{ inputs.app }} --name ${{ inputs.app }} + ./create.py environment --site default --app ${{ inputs.app }} --name ${{ inputs.app }} spack env activate envs/${{ inputs.app }} # LLVM Clang not in PATH, search for it specifically @@ -149,7 +184,7 @@ runs: # So Spack can find external MPI export "PATH=$HOME/mpi/bin:${PATH}" - # No external find for intel-oneapi-mpi, and problems finding other MPI libraries as well + # No external find for intel-oneapi-mpi, # And no way to add object entry to list using "spack config add" # Add this first so "spack config add packages:" will append to this entry if [[ "${{ inputs.mpi }}" == "intel-oneapi-mpi" ]]; then @@ -163,6 +198,25 @@ runs: echo " prefix: /opt/intel/oneapi" >> ${SPACK_ENV}/spack.yaml fi + # Same for qt@5 on macOS or on Linux w/ Intel + if [[ "$RUNNER_OS" == "macOS" ]]; then + echo "" >> ${SPACK_ENV}/spack.yaml + echo " packages:" >> ${SPACK_ENV}/spack.yaml + echo " qt:" >> ${SPACK_ENV}/spack.yaml + echo " buildable: False" >> ${SPACK_ENV}/spack.yaml + echo " externals:" >> ${SPACK_ENV}/spack.yaml + echo " - spec: qt@5" >> ${SPACK_ENV}/spack.yaml + echo " prefix: /usr/local/opt/qt@5" >> ${SPACK_ENV}/spack.yaml + #elif [[ "${{ inputs.compiler }}" == "intel"* ]]; then + # echo "" >> ${SPACK_ENV}/spack.yaml + # echo " packages:" >> ${SPACK_ENV}/spack.yaml + # echo " qt:" >> ${SPACK_ENV}/spack.yaml + # echo " buildable: False" >> ${SPACK_ENV}/spack.yaml + # echo " externals:" >> ${SPACK_ENV}/spack.yaml + # echo " - spec: qt@5" >> ${SPACK_ENV}/spack.yaml + # echo " prefix: /usr" >> ${SPACK_ENV}/spack.yaml + fi + # Spack external find is by default only looking for build-tools. Either # search for additional packages explicitly, or fall back to the old # behavior to find all external packages. @@ -177,6 +231,10 @@ runs: source setup.sh spack env activate ${{ inputs.path }}/envs/${{ inputs.app }} + # Speed up builds using two build jobs per package and four + # jobs in parallel further down (spack install) + spack config add "config:build_jobs:2" + # Use external MPI to save compilation time spack config add "packages:mpi:buildable:False" spack config add "packages:all:providers:mpi:[${{ inputs.mpi }}]" @@ -193,6 +251,10 @@ runs: # Error when proj attempts to build using external on macOS spack config rm "packages:ncurses" + # Remove external m4 entry + # Error when cairo attempts to build using external on macOS + spack config rm "packages:m4" + spack config add "packages:all:compiler:[${{ inputs.compiler }}]" # Currently, this is the case for all workflows - wrap in if statement @@ -207,9 +269,6 @@ runs: # Clang has trouble finding -lomp for OpenMP because of non-standard location # Could be fixed by setting compiler environment LDFLAGS to -L$(brew --prefix llvm)/lib or wherever libomp is spack config add "packages:wgrib2:variants: ~openmp" - # DH* This should no longer be necessary - spack config add "packages:boost:version:[1.78.0]" - # *DH spack config add "packages:fms:variants: +64bit +enable_quad_precision +gfs_phys ~openmp +pic" spack config add "packages:fms-jcsda:variants: +64bit +enable_quad_precision +gfs_phys ~openmp +pic" fi @@ -225,11 +284,26 @@ runs: - name: build-env shell: bash + env: + OMP_NUM_THREADS: 16 run: | cd ${{ inputs.path }} source setup.sh spack env activate ${{ inputs.path }}/envs/${{ inputs.app }} - spack install --fail-fast + # DH* For now only with Intel to avoid having to rerun all CI tests + if [[ "${{ inputs.compiler }}" == "intel"* ]]; then + for i in {1..8}; do + nohup spack install --fail-fast >> spack_install.log 2>&1 & + done + # + while ps -ef | grep -ve 'grep' | grep 'spack install'; do + echo "Still running" + tail -n 10 spack_install.log + sleep 60 + done + else + spack install --fail-fast + fi - name: create-meta-modules shell: bash diff --git a/.github/workflows/create-spack-mirror.yaml b/.github/workflows/create-spack-mirror.yaml index 1f5fc8819..f63e9751a 100644 --- a/.github/workflows/create-spack-mirror.yaml +++ b/.github/workflows/create-spack-mirror.yaml @@ -26,7 +26,7 @@ jobs: - name: create-env run: | source ./setup.sh - ./create-env.py --app ${{ github.event.inputs.app }} --site ${{ github.event.inputs.site }} --name mirror + ./create.py environment --app ${{ github.event.inputs.app }} --site ${{ github.event.inputs.site }} --name mirror spack env activate -d envs/mirror spack compiler find spack add re2c diff --git a/README.md b/README.md index 54bcae38a..5019651e0 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ spack-stack is a collaborative effort between the NOAA Environmental Modeling Ce [spack](https://github.com/spack/spack) is a community-supported, multi-platform, Python-based package manager originally developed by the Lawrence Livermore National Laboratory (LLNL; https://computing.llnl.gov/projects/spack-hpc-package-manager). It is provided as a submodule so that a stable version can be referenced. [See the Spack Documentation for more information](https://spack.readthedocs.io/en/latest/) spack-stack is mainly a collection of Spack configuration files, but provides a few Python scripts to simplify the installation process: -- `create-env.py` is provided to copy common, site-specific, and application-specific configuration files into a coherent Spack environment -- `meta_modules/setup_meta_modules.py` creates compiler, MPI and Python meta-modules for a convenient setup of a user environment using modules (currently lua only) +- `create.py` is provided to copy common, site-specific, and application-specific configuration files into a coherent Spack environment and to create container recipes +- `meta_modules/setup_meta_modules.py` creates compiler, MPI and Python meta-modules for a convenient setup of a user environment using modules (lua and tcl) spack-stack is maintained by: - Kyle Gerheiser (@kgerheiser), NOAA-EMC @@ -32,7 +32,6 @@ For questions or problems, please consult the currently open [issues](https://gi **Note. spack-stack is in early development and not yet ready for use. Instructions may be incomplete or invalid.** ## Quickstart - ``` git clone https://github.com/NOAA-EMC/spack-stack.git cd spack-stack @@ -42,12 +41,18 @@ cd spack-stack # Sources Spack from submodule and sets ${SPACK_STACK_DIR} source setup.sh +# Basic usage of create.py +./create.py -h +``` + +### Create local environment +``` # See a list of sites and apps -./create-env.py -h +./create.py environment -h -# Creates a pre-configured Spack environment in envs/. -# Copies site-specific, application-specific, and common config files into the environment directory -./create-env.py --site hera --app jedi-fv3 --name jedi-fv3.hera +# Create a pre-configured Spack environment in envs/. +# (copies site-specific, application-specific, and common config files into the environment directory) +./create.py environment --site hera --app jedi-fv3 --name jedi-fv3.hera # Activate the newly created environment # Optional: decorate the command line prompt using -p @@ -57,10 +62,9 @@ source setup.sh spack env activate [-p] envs/jedi-fv3.hera # Optionally edit config files (spack.yaml, packages.yaml compilers.yaml, site.yaml) -cd envs/jedi-fv3.hera -emacs spack.yaml -emacs common/*.yaml -emacs site/*.yaml +emacs envs/jedi-fv3.hera/spack.yaml +emacs envs/jedi-fv3.hera/common/*.yaml +emacs envs/jedi-fv3.hera/site/*.yaml # Process the specs and install # Note: both steps will take some time! @@ -74,6 +78,53 @@ spack module lmod refresh ./meta_modules/setup_meta_modules.py ``` +### Create container +``` +# See a list of preconfigured containers +./create.py container -h + +# Create container spack definition (spack.yaml) in directory envs/. +./create.py container --config docker-ubuntu-gcc-openmpi --spec esmf + +# Descend into container environment directory +cd envs/esmf.docker-ubuntu-gcc-openmpi + +# Optionally edit config file +emacs spack.yaml + +# Docker: create Dockerfile and build container +# See section "container" in spack.yaml for additional information +spack containerize > Dockerfile +docker build -t myimage . +docker run -it myimage +``` + +## Generating new site config +Recommended: Start with an empty (default) site config. Then run `spack external find` to locate common external packages such as git, Perl, CMake, etc., and run `spack compiler find` to locate compilers in your path. Compilers or external packages with modules need to be added manually. +``` +./create.py environment --site default --app jedi-ufs --name jedi-ufs.mysite + +# Descend into site config directory +cd envs/jedi-ufs.mysite/site + +# Find external packages and compilers, output the files here +# (overwrites packages.yaml and compilers.yaml) +SPACK_SYSTEM_CONFIG_PATH=`pwd` spack external find --all --scope system +SPACK_SYSTEM_CONFIG_PATH=`pwd` spack compiler find --scope system + +# Optionally edit config files as above in the quickstart section + +# Optionally attempt to find additional packages by name, +# for example: "spack external find wget" +``` +It is also instructive to peruse the GitHub actions scripts in `.github/workflows` and `.github/actions` to see how automated spack-stack builds are configured for CI testing, as well as the existing site configs in `configs/sites`. + +## Known issues + +### General +1. First call to `spack concretize` fails with `[Errno 2] No such file or directory: ... .json` +This can happen when `spack concretize` is called the very first time in a new spack-stack clone, during which the boostrapping (installation of clingo) is done first. Simply rerunning the command should solve the problem. + ## Disclaimer The United States Department of Commerce (DOC) GitHub project code is diff --git a/configs/apps/nceplibs/spack.yaml b/configs/apps/nceplibs/spack.yaml index 45914db68..de613a244 100644 --- a/configs/apps/nceplibs/spack.yaml +++ b/configs/apps/nceplibs/spack.yaml @@ -6,4 +6,4 @@ spack: @CONFIG_INCLUDES@ specs: - - nceplibs-bundle + - nceplibs-bundle-env diff --git a/configs/common/README.md b/configs/common/README.md deleted file mode 100644 index 1d9e293e0..000000000 --- a/configs/common/README.md +++ /dev/null @@ -1,4 +0,0 @@ -### Known issues - -1. First call to `spack concretize` fails with `[Errno 2] No such file or directory: ... .json` -This can happen when `spack concretize` is called the very first time in a new spack-stack clone, during which the boostrapping (installation of clingo) is done first. Simply rerunning the command should slove the problem. diff --git a/configs/common/config.yaml b/configs/common/config.yaml index 2f932d06f..b301153fb 100644 --- a/configs/common/config.yaml +++ b/configs/common/config.yaml @@ -26,3 +26,6 @@ config: # Cache directory for miscellaneous files, like the package index. # This can be purged with `spack clean --misc-cache` misc_cache: ${SPACK_STACK_DIR}/cache/misc_cache + + # Timeout in seconds for web requests (curl/urllib) + connect_timeout: 60 diff --git a/configs/common/modules.yaml b/configs/common/modules.yaml index e9b9152ee..9b84eea71 100644 --- a/configs/common/modules.yaml +++ b/configs/common/modules.yaml @@ -9,6 +9,7 @@ modules: projections: all: '{compiler.name}/{compiler.version}/{name}/{version}' ^mpi: '{^mpi.name}/{^mpi.version}/{compiler.name}/{compiler.version}/{name}/{version}' + ecmwf-atlas: '{compiler.name}/{compiler.version}/atlas/{version}' nlohmann-json: '{compiler.name}/{compiler.version}/json/{version}' nlohmann-json-schema-validator: '{compiler.name}/{compiler.version}/json-schema-validator/{version}' blacklist: @@ -95,6 +96,10 @@ modules: '{name}_ROOT': '{prefix}' suffixes: +debug: 'debug' + ecmwf-atlas: + environment: + set: + 'atlas_ROOT': '{prefix}' esmf: environment: set: @@ -107,6 +112,10 @@ modules: suffixes: ^esmf@8.2.0~debug: 'esmf-8.2.0' ^esmf@8.2.0+debug: 'esmf-8.2.0-debug' + openmpi: + environment: + set: + 'OMPI_MCA_rmaps_base_oversubscribe': '1' lmod: # Core compiler is a dummy that is not used to build the stack, do not change. @@ -114,6 +123,7 @@ modules: core_compilers:: - gcc@4.6 projections: + ecmwf-atlas: 'atlas/{version}' nlohmann-json: 'json/{version}' nlohmann-json-schema-validator: 'json-schema-validator/{version}' blacklist: @@ -200,6 +210,10 @@ modules: '{name}_ROOT': '{prefix}' suffixes: +debug: 'debug' + ecmwf-atlas: + environment: + set: + 'atlas_ROOT': '{prefix}' esmf: environment: set: @@ -212,6 +226,10 @@ modules: suffixes: ^esmf@8.2.0~debug: 'esmf-8.2.0' ^esmf@8.2.0+debug: 'esmf-8.2.0-debug' + openmpi: + environment: + set: + 'OMPI_MCA_rmaps_base_oversubscribe': '1' hierarchy: - mpi diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 32683059a..6bce21bfb 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -8,6 +8,7 @@ yacc:: [bison] gl:: [opengl] glu:: [openglu] + jpeg:: [libjpeg-turbo] bison: version: [3.8.2] wget: @@ -18,6 +19,9 @@ version: [2.6.3] zlib: version: [1.2.11] + hdf: + version: [4.2.15] + variants: ~fortran +netcdf hdf5: version: [1.12.1] variants: +hl +fortran +mpi +threadsafe @@ -135,7 +139,7 @@ fckit: version: [0.9.5] variants: +eckit - atlas: + ecmwf-atlas: version: [0.27.0] variants: +fckit shumlib: @@ -145,8 +149,13 @@ ectrans: version: [develop] variants: ~enable_mkl - git-lfs: - version: [2.11.0] + #git-lfs: + # Assume git-lfs is provided, hard to install + # because of dependencies on go/go-bootstrap. + # No need to uncomment here as long as the + # site config provides an external package. + #buildable: False + #version: [2.11.0] openblas: version: [0.3.19] variants: +noavx512 @@ -168,6 +177,8 @@ variants: +blas +lapack py-pybind11: version: [2.8.1] + py-pyhdf: + version: [0.10.4] py-h5py: version: [3.6.0] py-netcdf4: @@ -208,9 +219,9 @@ udunits: version: [2.2.28] openmpi: - variants: +internal-hwloc + variants: +internal-hwloc +two_level_namespace mpich: - variants: ~hwloc + variants: ~hwloc +two_level_namespace met: version: [10.1.0] variants: +python +grib2 diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml new file mode 100644 index 000000000..42e9d3618 --- /dev/null +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -0,0 +1,118 @@ +spack: + concretization: together + view: false + + config: + build_jobs: 2 + connect_timeout: 60 + + compilers: + - compiler: + spec: gcc@10 + paths: + cc: /usr/bin/gcc-10 + cxx: /usr/bin/g++-10 + f77: /usr/bin/gfortran-10 + fc: /usr/bin/gfortran-10 + flags: {} + operating_system: any + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + + # Basic package config from configs/common/packages.yaml + @PACKAGE_CONFIG@ + # Additional package config for container + gcc: + buildable: False + externals: + - spec: gcc@10 + prefix: /usr + git: + buildable: False + externals: + - spec: git@2 + prefix: /usr + git-lfs: + buildable: False + externals: + - spec: git-lfs@2 + prefix: /usr + + specs: + ### Not yet tested + #- jedi-ewok-env + #- jedi-um-bundle-env + #- jedi-tools-env + #- ufs-weather-model-env + #- nceplibs-bundle-env + ### Tested to work - increasing level of complexity + #- wget + #- esmf + #- base-env + #- jedi-base-env + #- jedi-fv3-bundle-env + #- jedi-ufs-bundle-env + #- jedi-um-bundle-env + - @SPEC@ + + container: + + # Select the format of the recipe e.g. docker, + # singularity or anything else that is currently supported + format: docker + # How to use: + #$ spack containerize > Dockerfile + #$ docker build -t myimage . + #$ docker run -it myimage + + #format: singularity + # How to use: + #$ spack containerize > hdf5.def + #$ sudo singularity build hdf5.sif hdf5.def + + # Sets the base images for the stages where Spack builds the + # software or where the software gets installed after being built.. + images: + os: ubuntu:20.04 + spack: + url: https://github.com/noaa-emc/spack + ref: jcsda_emc_spack_stack + resolve_sha: false + + # Whether or not to strip binaries + strip: true + + ## Additional system packages that are needed at runtime + os_packages: + build: + - gcc-10 + - g++-10 + - gfortran-10 + - cpp-10 + #- libgomp + - git + - git-lfs + final: + - gcc-10 + - g++-10 + - gfortran-10 + - cpp-10 + #- libgomp + - git + - git-lfs + + ## Extra instructions + #extra_instructions: + # build: + # # Needed for gfortran-11 + # RUN add-apt-repository ppa:ubuntu-toolchain-r/test + # RUN apt-get install gfortran-11 + # final: | + # RUN echo 'export PS1="\[$(tput bold)\]\[$(tput setaf 1)\][gromacs]\[$(tput setaf 2)\]\u\[$(tput sgr0)\]:\w $ "' >> ~/.bashrc + + # Labels for the image + labels: + app: "@SPEC@" + mpi: "openmpi" diff --git a/configs/repos/common/packages/base-env/package.py b/configs/repos/common/packages/base-env/package.py deleted file mode 100644 index 6ee64f9b2..000000000 --- a/configs/repos/common/packages/base-env/package.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -import os -import sys - -from spack import * - -class BaseEnv(BundlePackage): - """Basic development environment used by other environments""" - - homepage = "https://github.com/noaa-emc/spack-stack" - git = "https://github.com/noaa-emc/spack-stack.git" - - maintainers = ['climbfuji', 'kgerheiser', 'rhoneyager'] - - version('main', branch='main') - - # Basic utilities - if sys.platform == 'darwin': - depends_on('libbacktrace', type='run') - depends_on('cmake', type='run') - depends_on('git', type='run') - depends_on('wget', type='run') - depends_on('curl', type='run') - - # I/O - depends_on('zlib', type='run') - depends_on('szip', type='run') - depends_on('hdf5', type='run') - depends_on('netcdf-c', type='run') - depends_on('netcdf-fortran', type='run') - depends_on('parallel-netcdf', type='run') - depends_on('parallelio', type='run') - depends_on('nccmp', type='run') - - # Python - depends_on('python@3.7:') diff --git a/configs/repos/common/packages/gsl-lite/package.py b/configs/repos/common/packages/gsl-lite/package.py deleted file mode 100644 index 921dd0f88..000000000 --- a/configs/repos/common/packages/gsl-lite/package.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -# ---------------------------------------------------------------------------- -# If you submit this package back to Spack as a pull request, -# please first remove this boilerplate and all FIXME comments. -# -# This is a template package file for Spack. We've put "FIXME" -# next to all the things you'll want to change. Once you've handled -# them, you can save this file and test your package like this: -# -# spack install gsl-lite -# -# You can edit this file again by typing: -# -# spack edit gsl-lite -# -# See the Spack documentation for more information on packaging. -# ---------------------------------------------------------------------------- - -from spack import * - - -class GslLite(CMakePackage): - """gsl-lite – A single-file header-only version of ISO C++ Guidelines Support Library (GSL) for C++98, C++11, and later""" - - homepage = "https://github.com/gsl-lite/gsl-lite" - git = "https://github.com/gsl-lite/gsl-lite.git" - url = "https://github.com/gsl-lite/gsl-lite/archive/refs/tags/v0.38.1.tar.gz" - - maintainers = ['kgerheiser', 'edwardhartnett', 'Hang-Lei-NOAA'] - - version('0.40.0', commit='d6c8af99a1d95b3db36f26b4f22dc3bad89952de') - version('0.39.0', commit='d0903fa87ff579c30f608bc363582e6563570342') - version('0.38.1', sha256='c2fa2315fff312f3897958903ed4d4e027f73fa44235459ecb467ad7b7d62b18') - version('0.38.0', sha256='5d25fcd31ea66dac9e14da1cad501d95450ccfcb2768fffcd1a4170258fcbc81') - version('0.37.0', sha256='a31d51b73742bb234acab8d2411223cf299e760ed713f0840ffed0dabe57ca38') - version('0.36.0', sha256='c052cc4547b33cedee6f000393a7005915c45c6c06b35518d203db117f75c71c') - version('0.34.0', sha256='a7d5b2672b78704ca03df9ef65bc274d8f8cacad3ca950365eef9e25b50324c5') - - depends_on('cmake') - - variant('tests', default=False) - variant('cuda_tests', default=False) - variant('examples', default=False) - variant('static_analysis_demos', default=False) - variant('cmake_export_package_registry', default=False) - variant('compat_header', default=False) - variant('legacy_headers', default=False) - - def cmake_args(self): - args = [ - self.define_from_variant('GSL_LITE_OPT_BUILD_TESTS', 'tests'), - self.define_from_variant('GSL_LITE_OPT_BUILD_CUDA_TESTS', 'cuda_tests'), - self.define_from_variant('GSL_LITE_OPT_BUILD_EXAMPLES', 'examples'), - self.define_from_variant('GSL_LITE_LOPT_BUILD_STATIC_ANALYSIS_DEMOS', - 'static_analysis_demos'), - self.define_from_variant('CMAKE_EXPORT_PACKAGE_REGISTRY', - 'cmake_export_package_registry'), - self.define_from_variant('GSL_LITE_OPT_INSTALL_COMPAT_HEADER', - 'compat_header'), - self.define_from_variant('GSL_LITE_OPT_INSTALL_LEGACY_HEADERS', - 'legacy_headers') - ] - return args diff --git a/configs/repos/common/packages/py-pygrib/package.py b/configs/repos/common/packages/py-pygrib/package.py deleted file mode 100644 index f45371d7b..000000000 --- a/configs/repos/common/packages/py-pygrib/package.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2010 Jeffrey Whitaker -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of -# this software and associated documentation files (the "Software"), to deal in the -# Software without restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the -# Software, and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN -# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -from spack import * - - -class PyPygrib(PythonPackage): - """A high-level interface to the ECWMF ECCODES C library for reading GRIB files.""" - - homepage = "https://github.com/jswhit/pygrib" - pypi = "pygrib/pygrib-2.1.4.tar.gz" - - version('2.1.4', sha256='951a409eb3233dd95839dd77c0dbe4d8cbed8f21a4015b1047dec9edec65f545') - - depends_on('eccodes', type=('build', 'run')) - - depends_on('py-setuptools', type=('build', 'run')) - depends_on('py-cython', type=('build', 'run')) - depends_on('py-numpy', type=('build', 'run')) - depends_on('py-proj', type=('run')) - - #depends_on('python@2.6:2.8,3.2:', type=('build', 'run'), when='@0.9.0') - #depends_on('python@2.6:2.8,3.3:', type=('build', 'run'), when='@0.10.0') - #depends_on('python@2.7:2.8,3.4:', type=('build', 'run'), when='@0.13.3') - #depends_on('python@2.7:2.8,3.5:', type=('build', 'run'), when='@0.17.2') - #depends_on('python@3.6:', type=('build', 'run'), when='@0.18.0') - - - diff --git a/configs/repos/common/repo.yaml b/configs/repos/common/repo.yaml deleted file mode 100644 index 3433d4655..000000000 --- a/configs/repos/common/repo.yaml +++ /dev/null @@ -1,2 +0,0 @@ -repo: - namespace: common diff --git a/configs/repos/jedi/packages/atlas/clang_include_array.patch b/configs/repos/jedi/packages/atlas/clang_include_array.patch deleted file mode 100644 index a9a5ac7ee..000000000 --- a/configs/repos/jedi/packages/atlas/clang_include_array.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- a/src/atlas/grid/detail/partitioner/CubedSpherePartitioner.h 2022-01-12 08:44:07.000000000 -0700 -+++ b/src/atlas/grid/detail/partitioner/CubedSpherePartitioner.h 2022-01-12 08:44:11.000000000 -0700 -@@ -11,7 +11,7 @@ - #pragma once - - #include -- -+#include - #include "atlas/grid/detail/partitioner/Partitioner.h" - - namespace atlas { diff --git a/configs/repos/jedi/packages/atlas/package.py b/configs/repos/jedi/packages/atlas/package.py deleted file mode 100644 index 8559e3963..000000000 --- a/configs/repos/jedi/packages/atlas/package.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -import os -from spack import * - -class Atlas(CMakePackage): - """A library for numerical weather prediction and climate modelling.""" - - homepage = "https://software.ecmwf.int/wiki/display/atlas" - git = "https://github.com/ecmwf/atlas.git" - url = "https://github.com/ecmwf/atlas/archive/0.22.1.tar.gz" - - maintainers = ['climbfuji', 'rhoneyager'] - - version('master', branch='master') - version('develop', branch='develop') - version('0.27.0', commit='d825fad7ab415558a81415914a0fc60da1d0295a', preferred=True) - version('0.26.0', commit='3ae6184a598a00fbc6b1a77c3c9d5d808f1c65ea') - version('0.25.0', commit='3c74adda4960723f237db936132888e3fd380154') - version('0.24.1', commit='36772b5a72f91e99b30756808d8cff6edb415b8f') - version('0.24.0', commit='071bbb18c1fe3eac9d19557cc4490995e0af5184') - version('0.23.0', commit='7e0a1251685e07a5dcccc84f4d9251d5a066e2ee') - version('0.22.1', commit='e55e9c72883d24e3ed4d4eaaae330825a2d77dd3') - version('0.22.0', commit='a70030278541d4c4e18ebf92b683951749d60049') - version('0.21.0', commit='b7728bb798b9891ce62e1034fa21c0bc33a30cab') - - depends_on('ecbuild', type=('build')) - depends_on('eckit') - depends_on('boost cxxstd=14 visibility=hidden', when='@0.26.0:', type=('build', 'run')) - variant('fckit', default=True) - depends_on('fckit', when='+fckit') - depends_on('python') - - patch('clang_include_array.patch', when='%apple-clang') - patch('clang_include_array.patch', when='%clang') - - variant('shared', default=True) - - variant('trans', default=False) - depends_on('ectrans', when='+trans') - #variant('cgal', default=False) - #depends_on('cgal', when='+cgal') - variant('eigen', default=True) - depends_on('eigen', when='+eigen') - variant('fftw', default=True) - depends_on('fftw-api', when='+fftw') - - def cmake_args(self): - res = [ - self.define_from_variant('ENABLE_FCKIT', 'fckit'), - self.define_from_variant('ENABLE_TRANS', 'trans'), - self.define_from_variant('ENABLE_EIGEN', 'eigen'), - self.define_from_variant('ENABLE_FFTW', 'fftw'), - "-DPYTHON_EXECUTABLE:FILEPATH=" + self.spec['python'].command.path, - ] - if '~shared' in self.spec: - res.append('-DBUILD_SHARED_LIBS=OFF') - return res - diff --git a/configs/repos/jedi/packages/ectrans/package.py b/configs/repos/jedi/packages/ectrans/package.py deleted file mode 100644 index 79d7e8963..000000000 --- a/configs/repos/jedi/packages/ectrans/package.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack import * - - -class Ectrans(CMakePackage): - """Ectrans is the global spherical Harmonics transforms library, extracted from the IFS. - It is using a hybrid of MPI and OpenMP parallelisation strategies. The package contains - both single- and double precision Fortran libraries (trans_sp, trans_dp), as well as a - C interface to the double-precision version (transi_dp).""" - - homepage = "https://github.com/ecmwf-ifs/ectrans" - git = "https://github.com/ecmwf-ifs/ectrans.git" - - maintainers = ['climbfuji'] - - version('develop', branch='develop', no_cache=True, preferred=True) - version('main', branch='main', no_cache=True, preferred=False) - - variant('mpi', default=True, description='Use MPI?') - variant('openmp', default=True, description='Use OpenMP?') - - #variant('double_precision', default=True, description='Support for double precision?') - #variant('single_precision', default=True, description='Support for single precision?') - - variant('mkl', default=False, description='Use MKL?') - variant('fftw', default=True, description='Use FFTW?') - - #variant('transi', default=True, description='Compile TransI C-interface to trans?') - - depends_on('ecbuild', type=('build')) - depends_on('mpi', when='+mpi') - depends_on('blas') - depends_on('lapack') - depends_on('fftw-api', when='+fftw') - depends_on('mkl', when='+mkl') - - depends_on('fiat~mpi', when='~mpi') - depends_on('fiat+mpi', when='+mpi') - - def cmake_args(self): - args = [ - self.define_from_variant('ENABLE_MPI', 'mpi'), - self.define_from_variant('ENABLE_OMP', 'openmp'), - #self.define_from_variant('ENABLE_DOUBLE_PRECISION', 'double_precision'), - #self.define_from_variant('ENABLE_SINGLE_PRECISION', 'single_precision'), - self.define_from_variant('ENABLE_FFTW', 'fftw'), - self.define_from_variant('ENABLE_MKL', 'mkl') - #self.define_from_variant('ENABLE_TRANSI', 'transi') - ] - - return args diff --git a/configs/repos/jedi/packages/fckit/package.py b/configs/repos/jedi/packages/fckit/package.py deleted file mode 100644 index f8ff26acb..000000000 --- a/configs/repos/jedi/packages/fckit/package.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -import os -from spack import * - -class Fckit(CMakePackage): - """A Fortran toolkit for interoperating Fortran with C/C++.""" - - homepage = "https://software.ecmwf.int/wiki/display/fckit" - git = "https://github.com/ecmwf/fckit.git" - url = "https://github.com/ecmwf/fckit/archive/0.9.0.tar.gz" - - maintainers = ['rhoneyager', 'climbfuji'] - - version('master', branch='master') - version('develop', branch='develop') - version('0.9.5', commit='7ec9cf8ad8b619a8319199e834171c11e111c888', preferred=True) - version('0.9.4', commit='a83cfe6d4ed22c954548dd7c31e1fbad3cd2f908') - version('0.9.3', commit='3f612e107682c61c0b6806ea3fc12e9509a90664') - version('0.9.2', commit='26439f09a421b29d745f4c4810d7d40f2820f5ec') - version('0.9.1', commit='0b2c04d29ff141d1963e21da2add2e70f01163ce') - version('0.9.0', commit='9cd993a524264e079ae260dbc89faea599e270fc') - version('0.8.0', commit='4cd749f1eeac64eece00adb50abd072ea14fa2b1') - version('0.7.0', commit='5a9ad884c087ae4c188a5937acf078514519778f') - - depends_on('mpi') - depends_on('python') - depends_on('ecbuild', type=('build')) - - variant('eckit', default=True) - depends_on('eckit+mpi', when='+eckit') - - variant('shared', default=True) - - def cmake_args(self): - res = [ - self.define_from_variant('ENABLE_ECKIT', 'eckit'), - '-DCMAKE_C_COMPILER=%s' % self.spec['mpi'].mpicc, - '-DCMAKE_CXX_COMPILER=%s' % self.spec['mpi'].mpicxx, - '-DCMAKE_Fortran_COMPILER=%s' % self.spec['mpi'].mpifc, - "-DPYTHON_EXECUTABLE:FILEPATH=" + self.spec['python'].command.path, - '-DFYPP_NO_LINE_NUMBERING=ON' - ] - if '~shared' in self.spec: - res.append('-DBUILD_SHARED_LIBS=OFF') - - if self.spec.satisfies('%intel') or self.spec.satisfies('%gcc'): - cxxlib = 'stdc++' - elif self.spec.satisfies('%clang') or self.spec.satisfies('%apple-clang'): - cxxlib = 'c++' - else: - raise InstallError("C++ library not configured for compiler") - res.append('-DECBUILD_CXX_IMPLICIT_LINK_LIBRARIES={}'.format(cxxlib)) - - return res - diff --git a/configs/repos/jedi/packages/fiat/package.py b/configs/repos/jedi/packages/fiat/package.py deleted file mode 100644 index 2c3d43c81..000000000 --- a/configs/repos/jedi/packages/fiat/package.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack import * - - -class Fiat(CMakePackage): - """FIAT (Fortran IFS and Arpege Toolkit) is a collection of selected - Fortran utility libraries, extracted from the IFS/Arpege model.""" - - homepage = "https://github.com/ecmwf-ifs/fiat" - git = "https://github.com/ecmwf-ifs/fiat.git" - - maintainers = ['climbfuji'] - - version('develop', branch='develop', no_cache=True, preferred=True) - version('main', branch='main', no_cache=True, preferred=False) - - variant('mpi', default=True, description='Use MPI?') - variant('openmp', default=True, description='Use OpenMP?') - variant('fckit', default=True, description='Use fckit?') - #variant('dr_hook_multi_precision_handles', default=False, - # description='Use deprecated single precision handles for DR_HOOK?') - #variant('warnings', default=True, description='Enable compiler warnings') - - depends_on('ecbuild', type=('build')) - depends_on('mpi', when='+mpi') - depends_on('eckit', when='+fckit') - depends_on('fckit', when='+fckit') - - def cmake_args(self): - args = [ - self.define_from_variant('ENABLE_OMP', 'openmp'), - self.define_from_variant('ENABLE_MPI', 'mpi'), - self.define_from_variant('ENABLE_FCKIT', 'fckit') - #self.define_from_variant('ENABLE_DR_HOOK_MULTI_PRECISION_HANDLES', - # 'dr_hook_multi_precision_handles') - #self.define_from_variant('ENABLE_WARNINGS, 'warnings') - ] - - return args diff --git a/configs/repos/jedi/packages/fms-jcsda/package.py b/configs/repos/jedi/packages/fms-jcsda/package.py deleted file mode 100644 index 52d2dbf29..000000000 --- a/configs/repos/jedi/packages/fms-jcsda/package.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack import * - - -class FmsJcsda(CMakePackage): - """GFDL's Flexible Modeling System (FMS) is a software environment - that supports the efficient development, construction, execution, - and scientific interpretation of atmospheric, oceanic, and climate - system models. - This version has been adapated by JCSDA and is only meant to be - used temporarily, until the JCSDA changes have found their way - back into the official repository.""" - - #homepage = "https://github.com/jcsda/fms" - #git = "https://github.com/jcsda/fms.git" - homepage = "https://github.com/climbfuji/fms" - git = "https://github.com/climbfuji/fms.git" - - maintainers = ['climbfuji'] - - version('release-stable', branch='feature/no-openmp-option_default_on', no_cache=True, preferred=True) - version('dev-jcsda', branch='dev/jcsda', no_cache=True) - - variant('64bit', default=True, description='64 bit?') - variant('gfs_phys', default=True, description='Use GFS Physics?') - variant('openmp', default=True, description='Use OpenMP?') - variant('enable_quad_precision', default=True, description='Enable quad precision?') - variant('pic', default=True, description='Generate position-independent code (PIC), useful ' - 'for building static libraries') - depends_on('mpi') - depends_on('netcdf-c') - depends_on('netcdf-fortran') - depends_on('ecbuild', type=('build'), when='@release-stable') - depends_on('jedi-cmake', type=('build'), when='@release-stable') - - def cmake_args(self): - args = [ - self.define_from_variant('64BIT'), - self.define_from_variant('GFS_PHYS'), - self.define_from_variant('OPENMP'), - self.define_from_variant('ENABLE_QUAD_PRECISION') - ] - - args.append(self.define('CMAKE_C_COMPILER', self.spec['mpi'].mpicc)) - args.append(self.define('CMAKE_CXX_COMPILER', self.spec['mpi'].mpicxx)) - args.append(self.define('CMAKE_Fortran_COMPILER', self.spec['mpi'].mpifc)) - - cflags = [] - fflags = [] - - if self.compiler.name in ['gcc', 'clang', 'apple-clang']: - gfortran_major_version = int(spack.compiler.get_compiler_version_output(self.compiler.fc, '-dumpversion').split('.')[0]) - if gfortran_major_version>=10: - fflags.append('-fallow-argument-mismatch') - - if '+pic' in self.spec: - cflags.append(self.compiler.cc_pic_flag) - fflags.append(self.compiler.fc_pic_flag) - - if cflags: - args.append(self.define('CMAKE_C_FLAGS', ' '.join(cflags))) - if fflags: - args.append(self.define('CMAKE_Fortran_FLAGS', ' '.join(fflags))) - - return args diff --git a/configs/repos/jedi/packages/jedi-base-env/package.py b/configs/repos/jedi/packages/jedi-base-env/package.py deleted file mode 100644 index 6720cd935..000000000 --- a/configs/repos/jedi/packages/jedi-base-env/package.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -import os -import sys - -from spack import * - -class JediBaseEnv(BundlePackage): - """Basic development environment for JEDI applications""" - - # Todo: update URL - homepage = "https://github.com/JCSDA-internal/jedi-stack" - git = "https://github.com/JCSDA-internal/jedi-stack.git" - - maintainers = ['climbfuji', 'rhoneyager'] - - version('main', branch='main') - - depends_on('base-env', type='run') - - depends_on('bison', type='run') - depends_on('flex', type='run') - - depends_on('netcdf-cxx4', type='run') - - depends_on('ecbuild', type='run') - depends_on('jedi-cmake', type='run') - - depends_on('git-lfs', type='run') - - depends_on('eigen', type='run') - depends_on('fftw-api', type='run') - depends_on('gsl-lite', type='run') - depends_on('udunits', type='run') - - depends_on('boost', type='run') - depends_on('blas', type='run') - depends_on('eckit', type='run') - depends_on('fckit', type='run') - depends_on('atlas', type='run') - depends_on('nlohmann-json', type='run') - depends_on('nlohmann-json-schema-validator', type='run') - - # Todo: check where all of this needs to be - jedi-base-env or one of the bundles? - depends_on('py-pandas', type='run') - depends_on('py-scipy', type='run') - depends_on('py-pybind11', type='run') - depends_on('py-h5py', type='run') - depends_on('py-netcdf4', type='run') - depends_on('py-pycodestyle', type='run') - depends_on('py-pyyaml', type='run') - depends_on('py-python-dateutil', type='run') - - depends_on('eccodes', type='run') - depends_on('py-eccodes', type='run') - - depends_on('bufr', type='run') diff --git a/configs/repos/jedi/packages/jedi-cmake/package.py b/configs/repos/jedi/packages/jedi-cmake/package.py deleted file mode 100644 index 06a3788c8..000000000 --- a/configs/repos/jedi/packages/jedi-cmake/package.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -import os -from spack import * - -class JediCmake(CMakePackage): - """CMake/ecbuild toolchains to facilitate portability on different systems.""" - - homepage = "https://github.com/JCSDA/jedi-cmake" - git = "https://github.com/JCSDA/jedi-cmake.git" - url = "https://github.com/JCSDA/jedi-cmake/archive/refs/tags/1.3.0.tar.gz" - - maintainers = ['climbfuji', 'rhoneyager'] - - version('master', branch='master', no_cache=True) - version('develop', branch='develop', no_cache=True) - version('1.3.0', sha256='b217e2250398f6c34f0da0a50a8efe500684af4d484adebaa87a9de630eee1b7', preferred=True) - version('1.2.0', sha256='eb9f1c403d1b43a90a5e774097382b183d56d5b40a1204b51af2da8db1559b21') - version('1.1.0', sha256='f1fe41eb5edd343bdf57eb76bea6d1b9f015878f0a9d0eb1e9dba18b903d3b35') - version('1.0.0', sha256='d773a800350e69372355b45e89160b593818cd438a86925b8a689c47996a0b9a') - - depends_on('cmake @3.10:', type=('build')) - diff --git a/configs/repos/jedi/packages/jedi-ewok-env/package.py b/configs/repos/jedi/packages/jedi-ewok-env/package.py deleted file mode 100644 index 4ebbdf5ea..000000000 --- a/configs/repos/jedi/packages/jedi-ewok-env/package.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -import os -import sys - -from spack import * - -class JediEwokEnv(BundlePackage): - """Development environment for ewok""" - - # DH* TODO UPDATE - homepage = "https://github.com/JCSDA-internal/ewok" - git = "https://github.com/JCSDA-internal/ewok.git" - - maintainers = ['climbfuji', '@ericlingerfelt'] - - version('main', branch='main') - - depends_on('base-env', type='run') - depends_on('jedi-base-env', type='run') - - depends_on('py-boto3', type='run') - depends_on('py-cartopy', type='run') - depends_on('py-jinja2', type='run') - depends_on('py-ruamel-yaml', type='run') - depends_on('py-ruamel-yaml-clib', type='run') - - depends_on('ecflow', type='run') - - conflicts('%gcc platform=darwin', msg='jedi-ewok-env does ' + \ - 'not build with gcc (11?) on macOS (12), use apple-clang') diff --git a/configs/repos/jedi/packages/jedi-fv3-bundle-env/package.py b/configs/repos/jedi/packages/jedi-fv3-bundle-env/package.py deleted file mode 100644 index c479278b7..000000000 --- a/configs/repos/jedi/packages/jedi-fv3-bundle-env/package.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -import os -import sys - -from spack import * - -class JediFv3BundleEnv(BundlePackage): - """Development environment for fv3-bundle""" - - homepage = "https://github.com/JCSDA-internal/fv3-bundle" - git = "https://github.com/JCSDA-internal/fv3-bundle.git" - - maintainers = ['climbfuji', 'rhoneyager'] - - version('main', branch='main') - - depends_on('base-env', type='run') - depends_on('jedi-base-env', type='run') - - depends_on('fms-jcsda@release-stable') diff --git a/configs/repos/jedi/packages/jedi-tools-env/package.py b/configs/repos/jedi/packages/jedi-tools-env/package.py deleted file mode 100644 index 3654e6024..000000000 --- a/configs/repos/jedi/packages/jedi-tools-env/package.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -import os -import sys - -from spack import * - -class JediToolsEnv(BundlePackage): - """Development environment for jedi-tools""" - - homepage = "https://github.com/JCSDA-internal/jedi-tools" - git = "https://github.com/JCSDA-internal/jedi-tools.git" - - maintainers = ['climbfuji', 'rhoneyager'] - - version('main', branch='main') - - depends_on('py-click', type='run') - depends_on('py-pandas', type='run') - depends_on('py-pygithub', type='run') - depends_on('py-openpyxl', type='run') - - conflicts('%intel', msg='jedi-tools-env does not build with Intel') diff --git a/configs/repos/jedi/packages/jedi-ufs-bundle-env/package.py b/configs/repos/jedi/packages/jedi-ufs-bundle-env/package.py deleted file mode 100644 index 3c58f7525..000000000 --- a/configs/repos/jedi/packages/jedi-ufs-bundle-env/package.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -import os -import sys - -from spack import * - -class JediUfsBundleEnv(BundlePackage): - """Development environment for fv3-bundle""" - - # DH* TODO - we should rename this to just ufs-bundle to match the other bundles - homepage = "https://github.com/JCSDA/ufs-jedi-bundle" - git = "https://github.com/JCSDA/ufs-jedi-bundle.git" - - maintainers = ['climbfuji', 'mark-a-potts'] - - version('main', branch='main') - - depends_on('base-env', type='run') - depends_on('jedi-base-env', type='run') - - depends_on('ufs-weather-model-env', type='run') diff --git a/configs/repos/jedi/packages/jedi-um-bundle-env/package.py b/configs/repos/jedi/packages/jedi-um-bundle-env/package.py deleted file mode 100644 index 2820d61b1..000000000 --- a/configs/repos/jedi/packages/jedi-um-bundle-env/package.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -import os -import sys - -from spack import * - -class JediUmBundleEnv(BundlePackage): - """Development environment for um-bundle""" - - # DH* TODO UPDATE - homepage = "https://github.com/JCSDA-internal/um-bundle" - git = "https://github.com/JCSDA-internal/um-bundle.git" - - maintainers = ['climbfuji', 'rhoneyager'] - - version('main', branch='main') - - depends_on('base-env', type='run') - depends_on('jedi-base-env', type='run') - - depends_on('shumlib', type='run') - depends_on('fiat', type='run') - depends_on('ectrans', type='run') diff --git a/configs/repos/jedi/packages/shumlib/package.py b/configs/repos/jedi/packages/shumlib/package.py deleted file mode 100644 index 07185873f..000000000 --- a/configs/repos/jedi/packages/shumlib/package.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import os -from spack import * - - -class Shumlib(MakefilePackage): - """shumlib - set of libraries which are used by the UK Met Office's Unified Model, that may - be of use to external tools or applications where identical functionality is desired""" - - homepage = "https://github.com/metomi/shumlib" - #git = "https://github.com/metomi/shumlib.git" - git = "https://github.com/climbfuji/shumlib.git" - url = "https://github.com/metomi/shumlib/archive/refs/tags/2021.10.1.zip" - - maintainers = [ 'matthewrmshin', 'climbfuji' ] - - version('macos_clang_port', commit='e5e5c9f23ce2656aacd75a884c26b01a5380752e') - #version('2021.10.1', commit='545874fba961deadf4b2758926be7c26f4c8dcb9') - #version('2021.07.1', commit='a4ea525ad3bf04684ef39b0241991a350e2b7241') - #version('2021.03.1', commit='58f599ce9cfb4bd47197125548a44039695fa7f1') - #version('2020.11.1', commit='58f599ce9cfb4bd47197125548a44039695fa7f1') - - def edit(self, spec, prefix): - env['LIBDIR_OUT'] = os.path.join(os.path.join(self.build_directory, 'spack-build')) # '/Users/heinzell/scratch/tmp-shumlib-inst' # prefix - #env['LIBDIR_ROOT'] = self.build_directory - - def build(self, spec, prefix): - # DH* TODO: SWITCH FOR DIFFERENT ARCHITECTURES - if spec.satisfies('%clang') or spec.satisfies('%apple-clang'): - os.system('make -f make/vm-x86-gfortran-clang.mk') - #elif spec.satisfies('%gcc'): - else: - os.system('make -f make/vm-x86-gfortran-gcc.mk') - - def install(self, spec, prefix): - install_tree(os.path.join(os.getenv('LIBDIR_OUT'), 'include'), prefix.include) - install_tree(os.path.join(os.getenv('LIBDIR_OUT'), 'lib'), prefix.lib) diff --git a/configs/repos/jedi/packages/soca-bundle-env/package.py b/configs/repos/jedi/packages/soca-bundle-env/package.py deleted file mode 100644 index 89fbe052f..000000000 --- a/configs/repos/jedi/packages/soca-bundle-env/package.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -import os -import sys - -from spack import * - -class SocaBundleEnv(BundlePackage): - """Development environment for soca-bundle""" - - # DH* TODO UPDATE - homepage = "https://github.com/JCSDA-internal/soca" - git = "https://github.com/JCSDA-internal/soca.git" - - maintainers = ['climbfuji', 'travissluka' ] - - version('main', branch='main') - - depends_on('base-env', type='run') - depends_on('jedi-base-env', type='run') - - depends_on('nco', type='run') diff --git a/configs/repos/jedi/repo.yaml b/configs/repos/jedi/repo.yaml deleted file mode 100644 index 322b04d6e..000000000 --- a/configs/repos/jedi/repo.yaml +++ /dev/null @@ -1,2 +0,0 @@ -repo: - namespace: jedi diff --git a/configs/repos/repos.yaml b/configs/repos/repos.yaml deleted file mode 100644 index c3cc6bcf9..000000000 --- a/configs/repos/repos.yaml +++ /dev/null @@ -1,4 +0,0 @@ -repos: -- common -- jedi -- ufs diff --git a/configs/repos/ufs/packages/gftl-shared/package.py b/configs/repos/ufs/packages/gftl-shared/package.py deleted file mode 100644 index 42c09c2c6..000000000 --- a/configs/repos/ufs/packages/gftl-shared/package.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack import * - - -class GftlShared(CMakePackage): - """ - Provides common gFTL containers of Fortran intrinsic types that - are encountered frequently. - """ - - homepage = "https://github.com/Goddard-Fortran-Ecosystem/gFTL-shared" - url = "https://github.com/Goddard-Fortran-Ecosystem/gFTL-shared/releases/download/v1.4.1/gFTL-shared-1.4.1.tar" - - maintainers = ['kgerheiser', 'edwardhartnett', 'Hang-Lei-NOAA'] - - depends_on('m4', type=('build', 'run')) - - version('1.4.1', - sha256='78a1c20fe75430df0e2abc5d324905cf52ad22346080b66925bca514d90ff94a', - url='https://github.com/Goddard-Fortran-Ecosystem/gFTL-shared/releases/download/v1.4.1/gFTL-shared-1.4.1.tar') - - version('1.3.6', - sha256='6a6d618581b0d15213b2700b15102921f557340b73c7710405525641824e8703', - url='https://github.com/Goddard-Fortran-Ecosystem/gFTL-shared/releases/download/v1.3.6/gFTL-shared-v1.3.6.tar') - - version('1.3.0', - sha256='c9e8090fb74900bbfd6cc9a0f75626180062be18d8671d48bb46dd087880e6b2', - url='https://github.com/Goddard-Fortran-Ecosystem/gFTL-shared/releases/download/v1.3.0/gFTL-shared-v1.3.0.tar') diff --git a/configs/repos/ufs/packages/mapl/esma_cmake_apple_m1.patch b/configs/repos/ufs/packages/mapl/esma_cmake_apple_m1.patch deleted file mode 100644 index 978069905..000000000 --- a/configs/repos/ufs/packages/mapl/esma_cmake_apple_m1.patch +++ /dev/null @@ -1,16 +0,0 @@ ---- a/ESMA_cmake/GNU.cmake 2022-01-27 09:49:56.000000000 -0700 -+++ b/ESMA_cmake/GNU.cmake 2022-01-27 09:51:54.000000000 -0700 -@@ -123,7 +123,13 @@ - set (GNU_NATIVE_ARCH "native") - set (PREFER_AVX128 "-mprefer-avx128") - set (NO_FMA "-mno-fma") -+elseif (${proc_decription} MATCHES "Apple M1" AND ${CMAKE_HOST_SYSTEM_PROCESSOR} MATCHES "x86_64") -+ # Rosetta 2 emulator of x86_64 architecture on Apple Silicon -+ set (GNU_TARGET_ARCH "westmere") -+ set (GNU_NATIVE_ARCH "native") -+ set (PREFER_AVX128 "-mprefer-avx128") -+ set (NO_FMA "-mno-fma") - else () - message(FATAL_ERROR "Unknown processor. Contact Matt Thompson") - endif () - diff --git a/configs/repos/ufs/packages/mapl/package.py b/configs/repos/ufs/packages/mapl/package.py deleted file mode 100644 index 21fb40c2e..000000000 --- a/configs/repos/ufs/packages/mapl/package.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import os - -from spack import * - - -class Mapl(CMakePackage): - """ - MAPL is a foundation layer of the GEOS architecture, whose - original purpose is to supplement the Earth System Modeling - Framework (ESMF). MAPL fills in missing capabilities of ESMF, - provides higher-level interfaces for common boiler-plate logic, - and enforces various componentization conventions across ESMF - gridded components within GEOS. - - """ - - homepage = "https://github.com/GEOS-ESM/MAPL" - url = "https://github.com/GEOS-ESM/MAPL/archive/refs/tags/v2.8.1.tar.gz" - - maintainers = ['kgerheiser', 'edwardhartnett', 'Hang-Lei-NOAA'] - - version('2.8.1', sha256='a7657d4c52a66c3a6663e436d2c2dd4dbb81addd747e1ace68f59843665eb739') - version('2.8.0', sha256='6da60a21ab77ecebc80575f25b756c398ef48f635ab0b9c96932a1d4ebd8b4a0') - version('2.7.3', sha256='e8cdc0816471bb4c42673c2fa66d9d749f5a18944cd31580a2d6fd6d961ba163') - version('2.7.2', sha256='8f123352c665c434a18ff87304a71a61fb3342919adcccfea2a40729992d9f93') - version('2.7.1', sha256='8239fdbebd2caa47a232c24927f7a91196704e35c8b7909e1bbbefccf0647ea6') - - resource( - name='esma_cmake', - git='https://github.com/GEOS-ESM/ESMA_cmake.git', - tag='v3.4.3') - - resource( - name='CMakeModules', - git='https://github.com/NOAA-EMC/CMakeModules.git', - tag='v1.2.0') - - # Patch to configure Apple M1 chip in x86_64 Rosetta 2 emulator mode - patch('esma_cmake_apple_m1.patch', when='@:2.8.1') - - variant('flap', default=False) - variant('pflogger', default=False) - variant('esma_gfe_namespace', default=True) - variant('shared', default=True) - variant('debug', default=False, description='Make a debuggable version of the library') - - depends_on('mpi') - depends_on('hdf5') - depends_on('netcdf-c') - depends_on('esmf~debug', when='~debug') - depends_on('esmf+debug', when='+debug') - depends_on('yafyaml') - depends_on('gftl-shared') - depends_on('ecbuild') - - def cmake_args(self): - dir = os.getcwd() - ecbuild_prefix = self.spec["ecbuild"].prefix - long_arg = ('-DCMAKE_MODULE_PATH={pwd}/ESMA_cmake;{pwd}/CMakeModules/Modules;' + - '{ecbuild_prefix}/share/ecbuild/cmake') - args = [ - self.define_from_variant('BUILD_WITH_FLAP', 'flap'), - self.define_from_variant('BUILD_WITH_PFLOGGER', 'pflogger'), - self.define_from_variant('ESMA_USE_GFE_NAMESPACE', 'esma_gfe_namespace'), - self.define_from_variant('BUILD_SHARED_MAPL', 'shared'), - long_arg.format(pwd=dir, ecbuild_prefix=ecbuild_prefix), - '-DCMAKE_Fortran_FLAGS=-ffree-line-length-none', - '-DCMAKE_C_COMPILER=%s' % self.spec['mpi'].mpicc, - '-DCMAKE_CXX_COMPILER=%s' % self.spec['mpi'].mpicxx, - '-DCMAKE_Fortran_COMPILER=%s' % self.spec['mpi'].mpifc - ] - - # Compatibility flags for gfortran - fflags = [] - if self.compiler.name in ['gcc', 'clang', 'apple-clang']: - fflags.append('-ffree-line-length-none') - gfortran_major_version = int(spack.compiler.get_compiler_version_output(self.compiler.fc, '-dumpversion').split('.')[0]) - if gfortran_major_version>=10: - fflags.append('-fallow-invalid-boz') - fflags.append('-fallow-argument-mismatch') - if fflags: - args.append(self.define('CMAKE_Fortran_FLAGS', ' '.join(fflags))) - - return args diff --git a/configs/repos/ufs/packages/met/openmp_shape_patch.patch b/configs/repos/ufs/packages/met/openmp_shape_patch.patch deleted file mode 100644 index 001b7fea8..000000000 --- a/configs/repos/ufs/packages/met/openmp_shape_patch.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- a/src/basic/vx_util/data_plane_util.cc 2022-03-18 22:29:42.000000000 -0400 -+++ b/src/basic/vx_util/data_plane_util.cc 2022-03-18 22:30:44.000000000 -0400 -@@ -253,7 +253,7 @@ - - #pragma omp parallel default(none) \ - shared(mlog, dp, frac_dp, width, wrap_lon, t) \ -- shared(use_climo, cmn, csd, vld_t, bad) \ -+ shared(use_climo, cmn, csd, vld_t, bad, shape) \ - private(x, y, n_vld, n_thr, gp, v) - { - diff --git a/configs/repos/ufs/packages/met/package.py b/configs/repos/ufs/packages/met/package.py deleted file mode 100644 index 6054a6221..000000000 --- a/configs/repos/ufs/packages/met/package.py +++ /dev/null @@ -1,156 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack import * - - -class Met(AutotoolsPackage): - """ - Statistical tool that matches up grids with either gridded analyses or point observations and applies configurable methods to compute statistics and diagnostics - """ - - homepage = "https://dtcenter.org/community-code/model-evaluation-tools-met" - url = "https://github.com/dtcenter/MET/releases/download/v10.1.0/met-10.1.0.20220314.tar.gz" - - maintainers = ['kgerheiser'] - - version('10.1.0', sha256='8d4c1fb2311d8481ffd24e30e407a1b1bc72a6add9658d76b9c323f1733db336') - version('10.0.1', sha256='8e965bb0eb8353229a730af511c5fa62bad9744606ab6a218d741d29eb5f3acd') - version('10.0.0', sha256='92f37c8bd83c951d86026cce294a16e4d3aa6dd41905629d0a729fa1bebe668a') - version('9.1.3', sha256='7356a5ad79ca961fd965cadd93a7bf6c73b3aa5fb1a01a932580b94e66d0d0c8') - - variant('openmp', default=True, description='Use OpenMP multithreading') - variant('grib2', default=False, - description='Enable compilation of utilities using GRIB2') - variant('python', default=False, description='Enable python embedding') - variant('lidar2nc', default=False, - description='Enable compilation of lidar2nc') - variant('modis', default=False, description='Enable compilation of modis') - variant('graphics', default=False, - description='Enable compilation of mode_graphics') - - depends_on('gsl') - depends_on('bufr') - depends_on('zlib') - depends_on('netcdf-c') - depends_on('netcdf-cxx4') - depends_on('g2c', when='+grib2') - - depends_on('hdf-eos2', when='+modis') - depends_on('hdf-eos2', when='+lidar2nc') - depends_on('hdf', when='+modis') - depends_on('hdf', when='+lidar2nc') - - depends_on('cairo', when='+graphics') - depends_on('freetype', when='+graphics') - - depends_on('python@3.6.3:', when='+python', type=('build', 'run')) - depends_on('py-netcdf4', when='+python', type=('run')) - depends_on('py-numpy', when='+python', type=('run')) - depends_on('py-xarray', when='+python', type=('run')) - depends_on('py-pandas', when='+python', type=('run')) - depends_on('py-cartopy', when='+python', type=('run')) - depends_on('py-matplotlib', when='+python', type=('run')) - depends_on('py-python-dateutil', when='+python', type=('run')) - - patch('openmp_shape_patch.patch', when='@10.1.0') - - def url_for_version(self, version): - release_date = { - '10.1.0': '20220314', - '10.0.1': '20211201', - '10.0.0': '20210510', - '9.1.3': '20210319' - } - url = "https://github.com/dtcenter/MET/releases/download/v{0}/met-{0}.{1}.tar.gz" - return url.format(version, release_date[str(version)]) - - def setup_build_environment(self, env): - spec = self.spec - cppflags = [] - ldflags = [] - libs = [] - - gsl = spec['gsl'] - env.set('MET_GSL', gsl.prefix) - - netcdfc = spec['netcdf-c'] - netcdfcxx = spec['netcdf-cxx4'] - zlib = spec['zlib'] - - cppflags.append('-I' + netcdfc.prefix.include) - cppflags.append('-I' + netcdfcxx.prefix.include) - cppflags.append('-D__64BIT__') - - ldflags.append('-L' + netcdfc.prefix.lib) - ldflags.append('-L' + netcdfcxx.prefix.lib) - ldflags.append('-L' + zlib.prefix.lib) - - libs.append('-lnetcdf') - libs.append('-lnetcdf_c++4') - libs.append('-lz') - - bufr = spec['bufr'] - bufr_libdir = find_libraries('libbufr_4', root=bufr.prefix, - shared=False, recursive=True).directories[0] - env.set('BUFRLIB_NAME', '-lbufr_4') - env.set('MET_BUFRLIB', bufr_libdir) - - if '+grib2' in spec: - g2c = spec['g2c'] - g2c_libdir = find_libraries('libg2c', root=g2c.prefix, - shared=False, recursive=True).directories[0] - env.set('MET_GRIB2CLIB', g2c_libdir) - env.set('GRIB2CLIB_NAME', '-lg2c') - - if '+python' in spec: - python = spec['python'] - env.set('MET_PYTHON', python.command.path) - env.set('MET_PYTHON_CC', '-I' + python.headers.directories[0]) - env.set('MET_PYTHON_LD', python.libs.ld_flags) - - if '+lidar2nc' in spec or '+modis' in spec: - hdf = spec['hdf'] - hdfeos = spec['hdf-eos2'] - env.set('MET_HDF5', hdf.prefix) - env.set('MET_HDFEOS', hdfeos.prefix) - - if '+graphics' in spec: - cairo = spec['cairo'] - freetype = spec['freetype'] - env.set('MET_CAIRO', cairo.prefix) - cppflags.append('-I' + cairo.prefix.include.cairo) - env.set('MET_FREETYPE', freetype.prefix) - - env.set('CPPFLAGS', ' '.join(cppflags)) - env.set('LIBS', ' '.join(libs)) - env.set('LDFLAGS', ' '.join(ldflags)) - - def configure_args(self): - args = [] - spec = self.spec - - if '+grib2' in spec: - args.append('--enable-grib2') - - if '+python' in spec: - args.append('--enable-python') - - if '~openmp' in spec: - args.append('--disable-openmp') - - if '+lidar2nc' in spec: - args.append('--enable-lidar2nc') - - if '+modis' in spec: - args.append('--enable-modis') - - if '+graphics' in spec: - args.append('--enable-mode_graphics') - - return args - - def setup_run_environment(self, env): - env.set('MET_BASE', self.prefix) diff --git a/configs/repos/ufs/packages/metplus/package.py b/configs/repos/ufs/packages/metplus/package.py deleted file mode 100644 index 291f35323..000000000 --- a/configs/repos/ufs/packages/metplus/package.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack import * -import os - - -class Metplus(Package): - """ - METplus is a verification framework that spans a wide range of temporal - (warn-on-forecast to climate) and spatial (storm to global) scales. - """ - - homepage = "https://dtcenter.org/community-code/metplus" - url = "https://github.com/dtcenter/METplus/archive/refs/tags/v4.1.0.tar.gz" - - maintainers = ['kgerheiser'] - - version('4.1.0', sha256='4e4d74be64c9c57b910824ebefff42eb3a9bb7e8e325d86b7a3f7fdd59d3e45d') - version('4.0.0', sha256='650c65b0cf1f1993209e69e469903c83fb4ae3c693060d8392fc1dece52493e2') - version('3.1.1', sha256='d137420c56b2736b09ab713300f25c16e1d6fe523d3f3e4d811471aed83b0d85') - - depends_on('met+python', type=('run')) - - def install(self, spec, prefix): - if spec.satisfies('@4.0.0:'): - conf = 'defaults.conf' - else: - conf = 'metplus_system.conf' - - metplus_config = FileFilter( - join_path('parm', 'metplus_config', conf)) - - met_prefix = spec['met'].prefix - metplus_config.filter(r'MET_INSTALL_DIR = /path/to', - 'MET_INSTALL_DIR = {}'.format(met_prefix)) - - install_tree(self.stage.source_path, prefix) - - def setup_run_environment(self, env): - env.prepend_path('PATH', self.prefix.ush) diff --git a/configs/repos/ufs/packages/nceplibs-bundle/package.py b/configs/repos/ufs/packages/nceplibs-bundle/package.py deleted file mode 100644 index c7ef8fe7f..000000000 --- a/configs/repos/ufs/packages/nceplibs-bundle/package.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -# ---------------------------------------------------------------------------- -# If you submit this package back to Spack as a pull request, -# please first remove this boilerplate and all FIXME comments. -# -# This is a template package file for Spack. We've put "FIXME" -# next to all the things you'll want to change. Once you've handled -# them, you can save this file and test your package like this: -# -# spack install nceplibs-bundle -# -# You can edit this file again by typing: -# -# spack edit nceplibs-bundle -# -# See the Spack documentation for more information on packaging. -# ---------------------------------------------------------------------------- - -from spack import * - - -class NceplibsBundle(BundlePackage): - """ - This is a collection of libraries commonly known as NCEPLIBS that are required - for several NCEP applications e.g. UFS, GSI, UPP, etc. - """ - - homepage = "https://github.com/NOAA-EMC/NCEPLIBS" - # There is no URL since there is no code to download. - - maintainers = ['kgerheiser', 'Hang-Lei-NOAA'] - - version('1.0.0') - - depends_on('bacio') - depends_on('bufr') - depends_on('crtm') - depends_on('g2') - depends_on('g2c') - depends_on('g2tmpl') - depends_on('gfsio') - depends_on('ip') - depends_on('landsfcutil') - depends_on('ncio') - depends_on('nemsio') - depends_on('sfcio') - depends_on('sigio') - depends_on('sp') - depends_on('w3emc') - depends_on('w3nco') - depends_on('wrf-io') - depends_on('wgrib2') - - # There is no need for install() since there is no code. diff --git a/configs/repos/ufs/packages/ufs-weather-model-debug-env/package.py b/configs/repos/ufs/packages/ufs-weather-model-debug-env/package.py deleted file mode 100644 index e54e11542..000000000 --- a/configs/repos/ufs/packages/ufs-weather-model-debug-env/package.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -import os -import sys - -from spack import * - -class UfsWeatherModelDebugEnv(BundlePackage): - """Development environment for ufs-weathermodel-bundle""" - - homepage = "https://github.com/ufs-community/ufs-weather-model" - git = "https://github.com/ufs-community/ufs-weather-model.git" - - maintainers = ['kgerheiser', 'climbfuji'] - - version('main', branch='main') - - depends_on('esmf+debug', type='run') - depends_on('mapl+debug', type='run') diff --git a/configs/repos/ufs/packages/ufs-weather-model-env/package.py b/configs/repos/ufs/packages/ufs-weather-model-env/package.py deleted file mode 100644 index 639bf7cb4..000000000 --- a/configs/repos/ufs/packages/ufs-weather-model-env/package.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -import os -import sys - -from spack import * - -class UfsWeatherModelEnv(BundlePackage): - """Development environment for ufs-weathermodel-bundle""" - - homepage = "https://github.com/ufs-community/ufs-weather-model" - git = "https://github.com/ufs-community/ufs-weather-model.git" - - maintainers = ['kgerheiser', 'climbfuji'] - - version('main', branch='main') - - depends_on('base-env', type='run') - - depends_on('esmf~debug', type='run') - depends_on('fms', type='run') - - depends_on('bacio', type='run') - depends_on('crtm', type='run') - depends_on('g2', type='run') - depends_on('g2tmpl', type='run') - depends_on('ip', type='run') - depends_on('sp', type='run') - depends_on('w3nco', type='run') - - depends_on('mapl~debug', type='run') diff --git a/configs/repos/ufs/packages/wgrib2/package.py b/configs/repos/ufs/packages/wgrib2/package.py deleted file mode 100644 index 1035ddf91..000000000 --- a/configs/repos/ufs/packages/wgrib2/package.py +++ /dev/null @@ -1,187 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -# ---------------------------------------------------------------------------- -# If you submit this package back to Spack as a pull request, -# please first remove this boilerplate and all FIXME comments. -# -# This is a template package file for Spack. We've put "FIXME" -# next to all the things you'll want to change. Once you've handled -# them, you can save this file and test your package like this: -# -# spack install wgrib -# -# You can edit this file again by typing: -# -# spack edit wgrib -# -# See the Spack documentation for more information on packaging. -# ---------------------------------------------------------------------------- - -import os -import sys -import re - -from spack import * - - -class Wgrib2(MakefilePackage): - """Utility for interacting with grib2 files""" - - homepage = "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2" - url = "https://www.ftp.cpc.ncep.noaa.gov/wd51we/wgrib2/wgrib2.tgz.v2.0.8" - - maintainers = ['kgerheiser', 'Hang-Lei-NOAA'] - - version('2.0.7', sha256='d7f1a4f9872922c62b3c7818c022465532cca1f5666b75d3ac5735f0b2747793', extension='tar.gz') - version('2.0.8', sha256='5e6a0d6807591aa2a190d35401606f7e903d5485719655aea1c4866cc2828160', extension='tar.gz') - version('3.1.0', sha256='5757ef9016b19ae87491918e0853dce2d3616b14f8c42efe3b2f41219c16b78f', extension='tar.gz') - version('3.1.1', sha256='9236f6afddad76d868c2cfdf5c4227f5bdda5e85ae40c18bafb37218e49bc04a', extension='tar.gz') - - variant('netcdf3', default=True, - description='Link in netcdf3 library to write netcdf3 files') - variant('netcdf4', default=False, - description='Link in netcdf4 library to write netcdf3/4 files') - variant('ipolates', default='3', - description='Link in IPOLATES library to interpolate to new grids (0=OFF, 1=ip, 3=ip2)', - values=('0', '1', '3')) - variant('spectral', default=False, - description='Spectral interpolation in -new_grid') - variant('fortran_api', default=True, - description='Make wgrib2api which allows fortran code to read/write grib2') - variant('mysql', default=False, - description='Link in interface to MySQL to write to mysql database') - variant('udf', default=False, - description='Add commands for user-defined functions and shell commands') - variant('regex', default=True, - description='Use regular expression library (POSIX-2') - variant('tigge', default=True, - description='Ability for TIGGE-like variable names') - variant('proj4', default=False, - description='The proj4 library is used to confirm that the gctpc code is working correctly') - variant('aec', default=True, - description='Enable use of the libaec library for packing with GRIB2 template') - variant('g2c', default=False, - description='include NCEP g2clib (mainly for testing purposes)') - variant('disable_timezone', default=False, - description='Some machines do not support timezones') - variant('disable_alarm', default=False, - description='Some machines do not support alarm(..) (not POSIX-1, IEEE Std 1003.1) use the alarm to terminate wgrib2 after N seconds') - variant('png', default=True, description='PNG encoding') - variant('jasper', default=True, description='JPEG compression using Jasper') - variant('openmp', default=True, description='OpenMP parallelization') - variant('wmo_validation', default=False, description='WMO validation') - - conflicts('+openmp', when='%apple-clang') - - variant_map = { - 'netcdf3': 'USE_NETCDF3', - 'netcdf4': 'USE_NETCDF4', - 'spectral': 'USE_SPECTRAL', - 'mysql': 'USE_MYSQL', - 'udf': 'USE_UDF', - 'regex': 'USE_REGEX', - 'tigge': 'USE_TIGGE', - 'proj4': 'USE_PROJ4', - 'aec': 'USE_AEC', - 'g2c': 'USE_G2CLIB', - 'png': 'USE_PNG', - 'jasper': 'USE_JASPER', - 'openmp': 'USE_OPENMP', - 'wmo_validation': 'USE_WMO_VALIDATION', - 'ipolates': 'USE_IPOLATES', - 'disable_timezone': 'DISABLE_TIMEZONE', - 'disable_alarm': 'DISABLE_ALARM', - 'fortran_api': 'MAKE_FTN_API' - } - - # Disable parallel build - parallel = False - - # Use Spack compiler wrapper flags - def inject_flags(self, name, flags): - if name == 'cflags': - if self.spec.compiler.name == 'apple-clang': - flags.append('-Wno-error=implicit-function-declaration') - - # When mixing Clang/gfortran need to link to -lgfortran - # Find this by searching for gfortran/../lib - if self.spec.compiler.name in ['apple-clang', 'clang']: - if 'gfortran' in self.compiler.fc: - output = Executable(self.compiler.fc)('-###', output=str, error=str) - libdir = re.search('--libdir=(.+?) ', output).group(1) - flags.append('-L{}'.format(libdir)) - - return (flags, None, None) - - flag_handler = inject_flags - - def url_for_version(self, version): - url = "https://www.ftp.cpc.ncep.noaa.gov/wd51we/wgrib2/wgrib2.tgz.v{}" - return url.format(version) - - def edit(self, spec, prefix): - makefile = FileFilter('makefile') - - # ifort no longer accepts -openmp - makefile.filter(r'-openmp', '-qopenmp') - makefile.filter(r'-Wall', ' ') - makefile.filter(r'-Werror=format-security', ' ') - - # clang doesn't understand --fast-math - if spec.satisfies('%clang') or spec.satisfies('%apple-clang'): - makefile.filter(r'--fast-math', '-ffast-math') - - for variant_name, makefile_option in self.variant_map.items(): - value = int(spec.variants[variant_name].value) - makefile.filter(r'^%s=.*' % makefile_option, - '{}={}'.format(makefile_option, value)) - - def setup_build_environment(self, env): - - if self.spec.compiler.name in 'intel': - comp_sys = 'intel_linux' - elif self.spec.compiler.name in ['gcc', 'clang', 'apple-clang']: - comp_sys = 'gnu_linux' - - env.set('COMP_SYS', comp_sys) - - def build(self, spec, prefix): - make() - - # Move wgrib2 executable to a tempoary directory - mkdir('install') - mkdir(join_path('install', 'bin')) - move(join_path('wgrib2', 'wgrib2'), join_path('install', 'bin')) - - # Build wgrib2 library by disabling all options - # and enabling only MAKE_FTN_API=1 - if '+fortran_api' in spec: - make('clean') - make('deep-clean') - makefile = FileFilter('makefile') - - # Disable all options - for variant_name, makefile_option in self.variant_map.items(): - value = 0 - makefile.filter(r'^%s=.*' % makefile_option, - '{}={}'.format(makefile_option, value)) - - # Enable MAKE_FTN_API to build library and USE_REGEX (there is a bug when off) - makefile.filter(r'^MAKE_FTN_API=.*', 'MAKE_FTN_API=1') - makefile.filter(r'^USE_REGEX=.*', 'USE_REGEX=1') - make('lib') - mkdir(join_path('install', 'lib')) - mkdir(join_path('install', 'include')) - - move(join_path('lib', 'libwgrib2.a'), - join_path('install', 'lib')) - move(join_path('lib', 'wgrib2api.mod'), - join_path('install', 'include')) - move(join_path('lib', 'wgrib2lowapi.mod'), - join_path('install', 'include')) - - def install(self, spec, prefix): - install_tree('install/', prefix) diff --git a/configs/repos/ufs/packages/yafyaml/package.py b/configs/repos/ufs/packages/yafyaml/package.py deleted file mode 100644 index 6e996359f..000000000 --- a/configs/repos/ufs/packages/yafyaml/package.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack import * - - -class Yafyaml(CMakePackage): - """ - yet another Fortran (implementation of) YAML - - The rationale for this one is simply to be compatible with the - containers in gFTL. It is not intended to be a complete YAML - parser, just the subset needed by my own projects. - - """ - - homepage = "https://github.com/Goddard-Fortran-Ecosystem/yaFyaml" - url = "https://github.com/Goddard-Fortran-Ecosystem/yaFyaml/archive/refs/tags/v0.5.1.tar.gz" - - maintainers = ['kgerheiser', 'edwardhartnett', 'Hang-Lei-NOAA'] - - version('0.5.1', sha256='7019460314e388b2d556db75d5eb734237a18494f79b921613addb96b7b7ce2f') - version('0.5.0', sha256='8ac5d41b1020e9311ac87f50dbd61b9f3e3188f3599ce463ad59650208fdb8ad') - - depends_on('gftl-shared') - - def cmake_args(self): - # FIXME: Add arguments other than - # FIXME: CMAKE_INSTALL_PREFIX and CMAKE_BUILD_TYPE - # FIXME: If not needed delete this function - args = [] - return args diff --git a/configs/repos/ufs/repo.yaml b/configs/repos/ufs/repo.yaml deleted file mode 100644 index 738871ea2..000000000 --- a/configs/repos/ufs/repo.yaml +++ /dev/null @@ -1,2 +0,0 @@ -repo: - namespace: ufs diff --git a/configs/sites/README.md b/configs/sites/README.md deleted file mode 100644 index 614f201bc..000000000 --- a/configs/sites/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Generating Site Configs - -``` -source ./setup.sh -mkdir && cd -# Output the files here -export SPACK_SYSTEM_CONFIG_PATH=`pwd` -spack external find --all --scope system -spack compiler find --scope system -``` - -## External Packages - -Run `spack external find` to locate common external packages such as git, Perl, CMake, etc. - -## Compilers - -Run `spack compiler find` to locate compilers in your path. Compilers with modules need to be added manually. diff --git a/configs/sites/macos-monterey-llvm-clang-openmpi-reference/README.md b/configs/sites/macos-monterey-apple-clang-openmpi-reference/README.md similarity index 88% rename from configs/sites/macos-monterey-llvm-clang-openmpi-reference/README.md rename to configs/sites/macos-monterey-apple-clang-openmpi-reference/README.md index 9fd26ce43..956a0b7a3 100644 --- a/configs/sites/macos-monterey-llvm-clang-openmpi-reference/README.md +++ b/configs/sites/macos-monterey-apple-clang-openmpi-reference/README.md @@ -26,8 +26,6 @@ This step is only required on the new arch64 systems that are equipped with a Ap ``` brew install coreutils@9.0 brew install gcc@11.2.0 -brew install llvm13.0.0 -brew install openmpi@4.1.2 brew install python@3.9.10 brew install git@2.34.1 brew install git-lfs@3.0.2 @@ -63,3 +61,5 @@ This error came up on macOS Monterey with mpich-3.4.3 installed via homebrew whe `CC=mpicc FC=mpif90 CXX=mpicxx` when calling `ecbuild` for those bundles. 2. Installation of `poetry` using `pip3` or test with `python3` fails This can happen when multiple versions of Python were installed with `brew` and `pip3`/`python3` point to different versions. Run `brew doctor` and check if there are issues with Python not being properly linked. Follow the instructions given by `brew`, if applicable. +3. Errors handling exceptions on macOS. A large number of errors handling exceptions thrown by applications was found when using default builds of mpich or openmpi, which use flat +namespaces. This is the case for both mpich and openmpi when installed via homebrew. With our spack version, mpich and openmpi can be installed with a `+two_level_namespace` option that fixes the problem (tested with mpich-3.4.2 and openmpi-4.1.3). \ No newline at end of file diff --git a/configs/sites/macos-monterey-gcc-openmpi-reference/compilers.yaml b/configs/sites/macos-monterey-apple-clang-openmpi-reference/compilers.yaml similarity index 96% rename from configs/sites/macos-monterey-gcc-openmpi-reference/compilers.yaml rename to configs/sites/macos-monterey-apple-clang-openmpi-reference/compilers.yaml index 2efb8aa86..0f6dd95fc 100644 --- a/configs/sites/macos-monterey-gcc-openmpi-reference/compilers.yaml +++ b/configs/sites/macos-monterey-apple-clang-openmpi-reference/compilers.yaml @@ -13,7 +13,7 @@ compilers:: environment: {} extra_rpaths: [] - compiler: - spec: gcc@11.2.0 + spec: gcc@11.3.0 paths: cc: /usr/local/bin/gcc-11 cxx: /usr/local/bin/g++-11 diff --git a/configs/sites/macos-monterey-gcc-openmpi-reference/config.yaml b/configs/sites/macos-monterey-apple-clang-openmpi-reference/config.yaml similarity index 100% rename from configs/sites/macos-monterey-gcc-openmpi-reference/config.yaml rename to configs/sites/macos-monterey-apple-clang-openmpi-reference/config.yaml diff --git a/configs/sites/macos-monterey-apple-clang-openmpi-reference/modules.yaml b/configs/sites/macos-monterey-apple-clang-openmpi-reference/modules.yaml new file mode 100644 index 000000000..b66a9a948 --- /dev/null +++ b/configs/sites/macos-monterey-apple-clang-openmpi-reference/modules.yaml @@ -0,0 +1,9 @@ +modules: + default: + enable:: + - lmod + lmod: + whitelist: + # List of packages for which we need modules that are blacklisted by default + - openmpi + - mpich diff --git a/configs/sites/macos-monterey-llvm-clang-openmpi-reference/packages.yaml b/configs/sites/macos-monterey-apple-clang-openmpi-reference/packages.yaml similarity index 85% rename from configs/sites/macos-monterey-llvm-clang-openmpi-reference/packages.yaml rename to configs/sites/macos-monterey-apple-clang-openmpi-reference/packages.yaml index 60cca45eb..3fa2deb01 100644 --- a/configs/sites/macos-monterey-llvm-clang-openmpi-reference/packages.yaml +++ b/configs/sites/macos-monterey-apple-clang-openmpi-reference/packages.yaml @@ -1,26 +1,23 @@ packages: all: - compiler:: [clang@13.0.0] + compiler:: [apple-clang@13.1.6] providers: - mpi:: [openmpi@4.1.2] + mpi:: [openmpi@4.1.3] ### MPI, Python, MKL - mpi: - buildable: False - openmpi: - externals: - - spec: openmpi@4.1.2~cuda~cxx~cxx_exceptions~java~memchecker+pmi+pmix~sqlite3~static~thread_multiple~wrapper-rpath - schedulers=slurm - prefix: /usr/local + # Build mpi provider as part of stack + #mpi: + # buildable: False python: buildable: False externals: # Use the homebrew version, since the native macOS version is # not suitable for development (missing python3-config, ...) - - spec: python@3.9.10+bz2+ctypes+dbm+ensurepip+lzma+nis+pyexpat~pythoncmd+readline+sqlite3+ssl~tix~tkinter+uuid+zlib + - spec: python@3.9.12+bz2+ctypes+dbm+ensurepip+lzma+nis+pyexpat~pythoncmd+readline+sqlite3+ssl~tix~tkinter+uuid+zlib prefix: /usr/local -### Modifications of common packages +### Modifications of common packages - need to turn off OpenMP with +### apple-clang or (llvm) clang; note: this is not needed with gcc fms: variants:: +64bit +enable_quad_precision +gfs_phys ~openmp +pic fms-jcsda: @@ -45,7 +42,7 @@ packages: prefix: /usr cmake: externals: - - spec: cmake@3.22.1 + - spec: cmake@3.23.1 prefix: /usr/local curl: externals: @@ -71,11 +68,11 @@ packages: externals: - spec: git@2.32.0~tcltk prefix: /usr - - spec: git@2.34.1~tcltk + - spec: git@2.36.0~tcltk prefix: /usr/local git-lfs: externals: - - spec: git-lfs@3.0.2 + - spec: git-lfs@3.1.4 prefix: /usr/local gmake: externals: @@ -101,6 +98,10 @@ packages: externals: - spec: ncurses@5.7.20081102 abi=5 prefix: /usr + opengl: + externals: + - spec: opengl@2.1 + prefix: /opt/X11 openssh: externals: - spec: openssh@8.6p1 diff --git a/configs/sites/macos-monterey-gcc-openmpi-reference/README.md b/configs/sites/macos-monterey-gcc-openmpi-reference/README.md deleted file mode 100644 index afe5be2e4..000000000 --- a/configs/sites/macos-monterey-gcc-openmpi-reference/README.md +++ /dev/null @@ -1,64 +0,0 @@ -# macOS - -## General instructions/prerequisites - -This instructions are meant to be a reference that users can follow to set up their own system. Depending on the user's setup and needs, some steps will differ, some may not be needed and others may be missing. Also, the package versions may change over time. This setup is valid as of 2022/03/17. - -### Install Apple's command line utilities -- Launch the Terminal, found in `/Applications/Utilities` -- Type the following command string: -``` -xcode-select --install -``` - -### Setup of x86_64 environment on arch64 systems -This step is only required on the new arch64 systems that are equipped with a Apple M1 silicon chip. -- Open `Applications` in Finder -- Duplicate your preferred terminal application (e.g. `Terminal` or `iTerm`) -- Rename the duplicate to, for example, `Terminal x86_64` -- Right-click / control+click on `Terminal x86_64`, choose `Get Info` -- Select the box `Open using Rosetta` and close the window - -### Install homebrew for x86_64 environment -- If your system is an arch64 system, make sure to open the newly created `Terminal x86_64` application. Type `arch` in the terminal to confirm, if correct the output is `i386` (and not `arm64`) -- Install `homebrew` from the command line. On x86_64 systems and on arch64 systems using the x86_64 emulator, `homebrew` is installed in `/usr/local` -- It is recommended to install the following prerequisites via Homebrew, as installing them with Spack and Apple's native clang can be tricky. -``` -brew install coreutils@9.0 -brew install gcc@11.2.0 -brew install openmpi@4.1.2 -brew install python@3.9.10 -brew install git@2.35.1 -brew install git-lfs@3.1.2 -brew install lmod@8.6.14 -brew install wget@1.21.3 -brew install bash@5.1.16 -brew install curl@7.82.0 -brew install cmake@3.22.3 -brew install openssl@1.1 -# Note - need to pin to version 5, not 6 -brew install qt@5.15.3 -``` -- Activate the `lua` module environment: -``` -source /usr/local/opt/lmod/init/profile -``` - -This environment enables working with spack and building new software environments, as well as loading modules that are created by spack for building JEDI and UFS software. - -### Temporary workaround for pip installs in spack -See https://github.com/spack/spack/issues/29308 -``` -which pip3 -# make sure this points to homebrew's pip3 -pip3 install poetry -# test - successful if no output -python3 -c "import poetry" -``` - -### Known issues -1. Error `invalid argument '-fgnu89-inline' not allowed with 'C++'` -This error came up on macOS Monterey with mpich-3.4.3 installed via homebrew when trying to build the jedi bundles that use `ecbuild`. The reason was that the C compiler flag `-fgnu89-inline` from `/usr/local/Cellar/mpich/3.4.3/lib/pkgconfig/mpich.pc` was added to the C++ compiler flags by ecbuild. The solution was to set -`CC=mpicc FC=mpif90 CXX=mpicxx` when calling `ecbuild` for those bundles. -2. Installation of `poetry` using `pip3` or test with `python3` fails -This can happen when multiple versions of Python were installed with `brew` and `pip3`/`python3` point to different versions. Run `brew doctor` and check if there are issues with Python not being properly linked. Follow the instructions given by `brew`, if applicable. diff --git a/configs/sites/macos-monterey-gcc-openmpi-reference/modules.yaml b/configs/sites/macos-monterey-gcc-openmpi-reference/modules.yaml deleted file mode 100644 index 4e9771f38..000000000 --- a/configs/sites/macos-monterey-gcc-openmpi-reference/modules.yaml +++ /dev/null @@ -1,4 +0,0 @@ -modules: - default: - enable:: - - lmod diff --git a/configs/sites/macos-monterey-gcc-openmpi-reference/packages.yaml b/configs/sites/macos-monterey-gcc-openmpi-reference/packages.yaml deleted file mode 100644 index a8363b027..000000000 --- a/configs/sites/macos-monterey-gcc-openmpi-reference/packages.yaml +++ /dev/null @@ -1,146 +0,0 @@ -packages: - all: - compiler:: [gcc@11.2.0] - providers: - mpi:: [openmpi@4.1.2] - -### MPI, Python, MKL - mpi: - buildable: False - openmpi: - externals: - - spec: openmpi@4.1.2~cuda~cxx~cxx_exceptions~java~memchecker+pmi+pmix~sqlite3~static~thread_multiple~wrapper-rpath - schedulers=slurm - prefix: /usr/local - python: - buildable: False - externals: - # Use the homebrew version, since the native macOS version is - # not suitable for development (missing python3-config, ...) - - spec: python@3.9.10+bz2+ctypes+dbm+ensurepip+lzma+nis+pyexpat~pythoncmd+readline+sqlite3+ssl~tix~tkinter+uuid+zlib - prefix: /usr/local - -### Modifications of common packages - -### All other external packages listed alphabetically - bash: - externals: - - spec: bash@5.1.16 - prefix: /usr/local - berkeley-db: - externals: - - spec: berkeley-db@4.7.25 - prefix: /usr - bison: - externals: - - spec: bison@2.3 - prefix: /usr - bzip2: - externals: - - spec: bzip2@1.0.8 - prefix: /usr - cmake: - externals: - - spec: cmake@3.22.3 - prefix: /usr/local - curl: - externals: - - spec: curl@7.82.0 - prefix: /usr/local/opt/curl - diffutils: - externals: - - spec: diffutils@2.8.1 - prefix: /usr - file: - externals: - - spec: file@5.41 - prefix: /usr - flex: - externals: - - spec: flex@2.6.4+lex - prefix: /usr - gettext: - externals: - - spec: gettext@0.21 - prefix: /usr/local - git: - externals: - - spec: git@2.32.0~tcltk - prefix: /usr - - spec: git@2.35.1~tcltk - prefix: /usr/local - git-lfs: - externals: - - spec: git-lfs@3.1.2 - prefix: /usr/local - gmake: - externals: - - spec: gmake@3.81 - prefix: /usr - groff: - externals: - - spec: groff@1.19.2 - prefix: /usr - hwloc: - externals: - - spec: hwloc@2.7.0 - prefix: /usr/local - krb5: - externals: - - spec: krb5@1.7-prerelease - prefix: /usr - m4: - externals: - - spec: m4@1.4.6 - prefix: /usr - ncurses: - externals: - - spec: ncurses@5.7.20081102 abi=5 - prefix: /usr - openssh: - externals: - - spec: openssh@8.6p1 - prefix: /usr - openssl: - externals: - - spec: openssl@1.1 - prefix: /usr/local/opt/openssl@1.1 - perl: - externals: - - spec: perl@5.18.4~cpanm+shared+threads - prefix: /usr - - spec: perl@5.30.3~cpanm+shared+threads - prefix: /usr - qt: - externals: - - spec: qt@5.15.3 - prefix: /usr/local/Cellar/qt@5/5.15.3 - rsync: - externals: - - spec: rsync@2.6.9 - prefix: /usr - ruby: - externals: - - spec: ruby@2.6.8 - prefix: /usr - # Don't use OS sqlite, this version doesn't have the development files - #sqlite: - # externals: - # - spec: sqlite@3.37.0+fts~functions+rtree - # prefix: /usr - texinfo: - externals: - - spec: texinfo@4.8 - prefix: /usr - wget: - externals: - - spec: wget@1.21.3 - prefix: /usr/local - xz: - externals: - - spec: xz@5.2.5 - prefix: /usr/local - zip: - externals: - - spec: zip@3.0 - prefix: /usr diff --git a/configs/sites/macos-monterey-llvm-clang-openmpi-reference/compilers.yaml b/configs/sites/macos-monterey-llvm-clang-openmpi-reference/compilers.yaml deleted file mode 100644 index 5f0e8f5da..000000000 --- a/configs/sites/macos-monterey-llvm-clang-openmpi-reference/compilers.yaml +++ /dev/null @@ -1,45 +0,0 @@ -compilers:: -- compiler: - spec: clang@13.0.0 - paths: - cc: /usr/local/opt/llvm/bin/clang - cxx: /usr/local/opt/llvm/bin/clang++ - f77: /usr/local/bin/gfortran - fc: /usr/local/bin/gfortran - flags: {} - operating_system: monterey - target: x86_64 - modules: [] - environment: - prepend_path: - PATH: '/usr/local/opt/llvm/bin' - CPATH: '/usr/local/opt/llvm/include' - LD_LIBRARY_PATH: '/usr/local/opt/llvm/lib' - extra_rpaths: - - '/usr/local/opt/llvm/lib' -- compiler: - spec: gcc@11.2.0 - paths: - cc: /usr/local/bin/gcc-11 - cxx: /usr/local/bin/g++-11 - f77: /usr/local/bin/gfortran-11 - fc: /usr/local/bin/gfortran-11 - flags: {} - operating_system: monterey - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: apple-clang@13.0.0 - paths: - cc: /usr/bin/clang - cxx: /usr/bin/clang++ - f77: /usr/local/bin/gfortran - fc: /usr/local/bin/gfortran - flags: {} - operating_system: monterey - target: any - modules: [] - environment: {} - extra_rpaths: [] diff --git a/configs/sites/macos-monterey-llvm-clang-openmpi-reference/config.yaml b/configs/sites/macos-monterey-llvm-clang-openmpi-reference/config.yaml deleted file mode 100644 index 84fe05ab9..000000000 --- a/configs/sites/macos-monterey-llvm-clang-openmpi-reference/config.yaml +++ /dev/null @@ -1,2 +0,0 @@ -config: - build_jobs: 8 diff --git a/configs/sites/macos-monterey-llvm-clang-openmpi-reference/modules.yaml b/configs/sites/macos-monterey-llvm-clang-openmpi-reference/modules.yaml deleted file mode 100644 index 4e9771f38..000000000 --- a/configs/sites/macos-monterey-llvm-clang-openmpi-reference/modules.yaml +++ /dev/null @@ -1,4 +0,0 @@ -modules: - default: - enable:: - - lmod diff --git a/create-env.py b/create-env.py deleted file mode 100755 index c6794c907..000000000 --- a/create-env.py +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -from operator import index -import os -from os import listdir, path, makedirs, linesep -import sys -import shutil -from argparse import RawTextHelpFormatter - -# Get directory of this script -stack_dir = path.dirname(path.realpath(__file__)) - -# Possible spack configuration files -valid_configs = ['compilers.yaml', 'config.yaml', 'mirrors.yaml', - 'modules.yaml', 'packages.yaml', 'repos.yaml', 'concretizer.yaml'] - -# Pass this value to --site for an empty config -empty_site = 'default' - -def stack_path(*paths): - return path.join(stack_dir, *paths) - - -def create_env_dir(name): - # Create spack-stack/envs to hold envrionments, if it doesn't exist - if not path.exists(stack_path('envs')): - makedirs(stack_path('envs')) - - # Create env_dir - env_dir = stack_path('envs', env_name) - if not path.exists(env_dir): - makedirs(env_dir) - print("Created environment {} in {}".format(env_name, env_dir)) - else: - sys.exit('Error: env {} already exists'.format(env_dir)) - - return env_dir - - -def copy_common_configs(env_dir): - common_dir = stack_path('configs', 'common') - common_configs = listdir(common_dir) - shutil.copytree(common_dir, path.join(env_dir, 'common')) - - -def copy_site_configs(site, env_dir): - site_dir = stack_path('configs', 'sites', site) - shutil.copytree(site_dir, path.join(env_dir, 'site')) - - -def copy_app_config(app, env_dir): - app_config = stack_path('configs', 'apps', app, 'spack.yaml') - with open(app_config, "r") as f: - contents = f.read() - - includes = [] - for config_type in ['site', 'common']: - config_dir = stack_path(env_dir, config_type) - if path.isdir(config_dir): - configs = list(filter(lambda f: f in valid_configs, listdir(config_dir))) - configs = map(lambda conf: ' - {}/{}'.format(config_type, conf), configs) - includes += configs - # Always include the repos (could be an option to point somewhere else) or not at all - includes += [' - ${SPACK_STACK_DIR}/configs/repos/repos.yaml'] - includes.insert(0, 'include:') if includes else includes.insert(0, '') - new_contents = contents.replace('@CONFIG_INCLUDES@', linesep.join(includes)) - - with open(path.join(env_dir, 'spack.yaml'), "w") as f: - f.write(new_contents) - - -def check_inputs(app, site): - app_path = stack_path('configs', 'apps', app, 'spack.yaml') - if not path.exists(app_path): - sys.exit('Error: invalid app {}, "{}" does not exist'.format(app, app_path)) - - if site == empty_site: - return - - site_path = stack_path('configs', 'sites', site) - if not path.exists(site_path): - sys.exit('Error: invalid site {}, "{}" does not exist'.format( - site, site_path)) - - -def site_help(): - _, site_dirs, _ = next(os.walk(stack_path('configs', 'sites'))) - help_string = 'Pre-configured platform, or "default" for an empty site.yaml.' + linesep - help_string += 'Defaults to "default" if no arg is given' + linesep - help_string += 'Available options are: ' + linesep - for site in site_dirs: - help_string += '\t' + site + linesep - return help_string - - -def app_help(): - _, app_dirs, _ = next(os.walk(stack_path('configs', 'apps'))) - help_string = 'Application environment to build.' + linesep - help_string += 'Available options are: ' + linesep - for app in app_dirs: - help_string += '\t' + app + linesep - return help_string - - -description_text = ''' - Create a pre-configured Spack environment. Envrionments are created in spack-stack/envs/. -''' - -epilog_text = """ -Example usage: - source setup.sh - ./create-env.py --site default --app ufs --name ufs-env - spack env activate [-p] envs/ufs-env - spack concretize - spack install -""" -parser = argparse.ArgumentParser(description=description_text, - epilog=epilog_text, - formatter_class=RawTextHelpFormatter) - -parser.add_argument('--site', type=str, required=False, nargs='?', default='default', help=site_help()) -parser.add_argument('--app', type=str, required=True, help=app_help()) -parser.add_argument('--name', type=str, required=False, - help='Optional name for env dir. Defaults to app.site name.') -parser.add_argument('--exclude-common-configs', required=False, - default=False, action='store_true', help='Ignore configs configs/common when creating environment') - -args = parser.parse_args() - -site = args.site -app = args.app -exclude_common_configs = args.exclude_common_configs -env_name = args.name if args.name else "{}.{}".format(app, site) - -check_inputs(app, site) -env_dir = create_env_dir(env_name) -if not exclude_common_configs: - copy_common_configs(env_dir) -copy_site_configs(site, env_dir) -copy_app_config(app, env_dir) diff --git a/create.py b/create.py new file mode 100755 index 000000000..c1b446608 --- /dev/null +++ b/create.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 + +import argparse +from operator import index +import os +from os import listdir, path, makedirs, linesep +import sys +import shutil +from argparse import RawTextHelpFormatter + +# Get directory of this script +stack_dir = path.dirname(path.realpath(__file__)) + +# Possible spack configuration files +valid_configs = ['compilers.yaml', 'config.yaml', 'mirrors.yaml', + 'modules.yaml', 'packages.yaml', 'concretizer.yaml'] + +# Pass this value to --site for an empty config +empty_site = 'default' + +def stack_path(*paths): + return path.join(stack_dir, *paths) + + +def create_env_dir(env_name): + # Create spack-stack/envs to hold envrionments, if it doesn't exist + if not path.exists(stack_path('envs')): + makedirs(stack_path('envs')) + + # Create env_dir + env_dir = stack_path('envs', env_name) + if not path.exists(env_dir): + makedirs(env_dir) + print("Created {} in {}".format(env_name, env_dir)) + else: + sys.exit('Error: {} already exists'.format(env_dir)) + + return env_dir + + +def copy_common_configs(env_dir): + common_dir = stack_path('configs', 'common') + common_configs = listdir(common_dir) + shutil.copytree(common_dir, path.join(env_dir, 'common')) + + +def copy_site_configs(site, env_dir): + site_dir = stack_path('configs', 'sites', site) + shutil.copytree(site_dir, path.join(env_dir, 'site')) + + +def copy_app_config(app, env_dir): + app_config = stack_path('configs', 'apps', app, 'spack.yaml') + with open(app_config, "r") as f: + contents = f.read() + + includes = [] + for config_type in ['site', 'common']: + config_dir = stack_path(env_dir, config_type) + if path.isdir(config_dir): + configs = list(filter(lambda f: f in valid_configs, listdir(config_dir))) + configs = map(lambda conf: ' - {}/{}'.format(config_type, conf), configs) + includes += configs + includes.insert(0, 'include:') if includes else includes.insert(0, '') + new_contents = contents.replace('@CONFIG_INCLUDES@', linesep.join(includes)) + + with open(path.join(env_dir, 'spack.yaml'), "w") as f: + f.write(new_contents) + + +def copy_container_config(container, spec, env_dir): + container_config = stack_path('configs', 'containers', container + '.yaml') + with open(container_config, "r") as f: + contents = f.read() + + common_package_config = stack_path('configs', 'common', 'packages.yaml') + with open(common_package_config, "r") as f: + common_package_contents = f.read() + # Replace double colons in common_package_contents with single colons + # due to a bug in spack that replaces '::' with ':":"' during concretization + common_package_contents = common_package_contents.replace('::', ':').rstrip('\n') + + new_contents = contents.replace('@PACKAGE_CONFIG@', common_package_contents) + new_contents = new_contents.replace('@SPEC@', spec) + with open(path.join(env_dir, 'spack.yaml'), "w") as f: + f.write(new_contents) + + +def check_inputs(app=None, site=None, container=None): + if app: + app_path = stack_path('configs', 'apps', app, 'spack.yaml') + if not path.exists(app_path): + sys.exit('Error: invalid app {}, "{}" does not exist'.format(app, app_path)) + if site: + if not site == empty_site: + site_path = stack_path('configs', 'sites', site) + if not path.exists(site_path): + sys.exit('Error: invalid site {}, "{}" does not exist'.format( + site, site_path)) + if container: + container_path = stack_path('configs', 'containers', container + '.yaml') + if not path.exists(container_path): + sys.exit('Error: invalid container config {}, "{}" does not exist'.format( + container, container_path)) + +def site_help(): + _, site_dirs, _ = next(os.walk(stack_path('configs', 'sites'))) + help_string = 'Pre-configured platform, or "default" for an empty site.yaml.' + linesep + help_string += 'Defaults to "default" if no arg is given' + linesep + help_string += 'Available options are: ' + linesep + for site in site_dirs: + help_string += '\t' + site + linesep + return help_string + + +def app_help(): + _, app_dirs, _ = next(os.walk(stack_path('configs', 'apps'))) + help_string = 'Application environment to build.' + linesep + help_string += 'Available options are: ' + linesep + for app in app_dirs: + help_string += '\t' + app + linesep + return help_string + + +def container_config_help(): + _, _, container_configs = next(os.walk(stack_path('configs', 'containers'))) + help_string = 'Pre-configured container.' + linesep + help_string += 'Available options are: ' + linesep + for config in container_configs: + help_string += '\t' + config.rstrip('.yaml') + linesep + return help_string + + +def spec_help(): + help_string = 'Any valid spack spec, e.g. "wget" or "jedi-ufs-bundle-env".' + linesep + return help_string + + +description_text = ''' + Create a pre-configured Spack environment or container in spack-stack/envs/. +''' + +epilog_text = """ +Example usage: + source setup.sh + # Option 1: To create a local environment + ./create.py environment --site default --app ufs [--name ufs-env] [--exclude-common-configs] + spack env activate [-p] envs/ufs-env + spack concretize + spack install + # Option 2: create a container + ./create.py container --site docker-ubuntu-gcc --app jedi-ufs + cd envs/jedi-ufs.docker-ubuntu-gcc + spack containerize > Dockerfile + docker build -t myimage . + docker run -it myimage +""" +parser = argparse.ArgumentParser(description=description_text, + epilog=epilog_text, + formatter_class=RawTextHelpFormatter) +subparsers = parser.add_subparsers(help='help for subcommand', dest='subcommand') + +env_parser = subparsers.add_parser('environment', help='Create local environment') +env_parser.add_argument('--site', type=str, required=False, nargs='?', default='default', help=site_help()) +env_parser.add_argument('--app', type=str, required=True, help=app_help()) +env_parser.add_argument('--name', type=str, required=False, + help='Optional name for env dir. Defaults to app.site name.') +env_parser.add_argument('--exclude-common-configs', required=False, + default=False, action='store_true', help='Ignore configs configs/common when creating environment') + +con_parser = subparsers.add_parser('container', help='Create container') +con_parser.add_argument('--config', type=str, required=True, help=container_config_help()) +con_parser.add_argument('--spec', type=str, required=True, help=spec_help()) + +args = parser.parse_args() + +create_env = False +create_con = False +if args.subcommand == 'environment': + create_env = True +elif args.subcommand == 'container': + create_con = True +else: + parser.print_help() + sys.exit(-1) + +if create_env: + site = args.site + app = args.app + exclude_common_configs = args.exclude_common_configs + env_name = args.name if args.name else "{}.{}".format(app, site) + + check_inputs(app=app, site=site) + env_dir = create_env_dir(env_name) + if not exclude_common_configs: + copy_common_configs(env_dir) + copy_site_configs(site, env_dir) + copy_app_config(app, env_dir) +else: + container = args.config + spec = args.spec + + check_inputs(container=container) + env_name = "{}.{}".format(spec, container) + env_dir = create_env_dir(env_name) + copy_container_config(container, spec, env_dir) \ No newline at end of file diff --git a/meta_modules/setup_meta_modules.py b/meta_modules/setup_meta_modules.py index e40e734d6..d98e21eeb 100755 --- a/meta_modules/setup_meta_modules.py +++ b/meta_modules/setup_meta_modules.py @@ -3,7 +3,7 @@ import copy import logging #logging.basicConfig(level=logging.INFO) -logging.basicConfig(level=logging.DEBUG) +logging.basicConfig(format='%(message)s', level=logging.DEBUG) import os import shutil import sys @@ -650,3 +650,5 @@ def substitute_config_vars(config_str): with open(python_module_file, 'w') as f: f.write(module_content) logging.info(" ... writing {}".format(python_module_file)) + +logging.info("Metamodule generation completed successfully in {}".format(meta_module_dir)) diff --git a/spack b/spack index 8b6a0c16d..4c5bb51e2 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 8b6a0c16dd983eff8e32b5aa10141c1bb64afa42 +Subproject commit 4c5bb51e2b425eca7b918bd19ee80f6cafe83ecf