From 9b0ee0060ef10efb885d90e3d57d93d706ff1454 Mon Sep 17 00:00:00 2001 From: Philippe Blain Date: Thu, 2 Jun 2022 11:26:36 -0400 Subject: [PATCH 1/6] machines: eccc: update our machine files (#725) A few notes: - We do need to request memory even on machines where we have excluseive node access. 20 GB was chosen rather arbitrarily. - We set umask to 022 to make the .o and .e files readable by group and others. - We use the minimal SSM package for the compiler and Intel MPI, but we keep the setup using environment modules commented if ever we need to weak things (i.e. I_MPI_LIBRARY_KIND) - We set OMP_STACKSIZE. Since d1e972a (Update OMP (CICE-Consortium#680), 2022-02-18), OpenMP threading is active in 'ice_transport_remap.F90', and the default OpenMP stack size needs to be adjusted to avoid stack overflows. We set it to a 64 Mb size as used for other machines. Also, remove dead code setting 'CICE_ACCT'. This variable was last used in 98e03077 (Update scripts, rename variables from CICE_ to ICE_ to be more reusable in icepack., 2017-09-15), and so did not do anything for any of the machines that were using it after that commit. Remove code in machines env files that was setting it based on '~/.cice_proj'. A few notes specific to 'gpsc3': - Since we use an '--export' directive to choose which environment variables are exported to the job environment by SLURM, SSMUSE_BASE and SSMUSE_PATH are not present in the environnement and loading domains without their full paths fails on csh, so use a full path. - We use the compiler package from main/opt instead of eccc/all/opt since we do not need the EC-specific variables to be set (and it also leads to job failures since BASE_ARCH is not defined). (cherry picked from commit c334aee2e27e27767a1758a0acf1a12fa69c8565) --- configuration/scripts/cice.batch.csh | 37 ++++++++++++++ configuration/scripts/cice.launch.csh | 37 ++++++++++++++ ...cros.millikan_intel => Macros.gpsc3_intel} | 33 +++++-------- ...{Macros.cesium_intel => Macros.ppp3_intel} | 33 +++++-------- .../scripts/machines/Macros.ppp5_intel | 49 +++++++++++++++++++ .../scripts/machines/Macros.ppp6_gnu | 44 +++++++++++++++++ .../scripts/machines/Macros.ppp6_gnu-impi | 44 +++++++++++++++++ .../scripts/machines/Macros.ppp6_intel | 49 +++++++++++++++++++ .../scripts/machines/Macros.ppp6_intel19 | 49 +++++++++++++++++++ .../scripts/machines/Macros.robert_intel | 49 +++++++++++++++++++ .../scripts/machines/Macros.underhill_intel | 49 +++++++++++++++++++ .../scripts/machines/env.cesium_intel | 28 ----------- configuration/scripts/machines/env.fram_intel | 5 -- .../scripts/machines/env.gpsc3_intel | 37 ++++++++++++++ .../scripts/machines/env.millikan_intel | 28 ----------- configuration/scripts/machines/env.ppp3_intel | 35 +++++++++++++ configuration/scripts/machines/env.ppp5_intel | 38 ++++++++++++++ configuration/scripts/machines/env.ppp6_gnu | 29 +++++++++++ .../scripts/machines/env.ppp6_gnu-impi | 38 ++++++++++++++ configuration/scripts/machines/env.ppp6_intel | 38 ++++++++++++++ .../scripts/machines/env.ppp6_intel19 | 39 +++++++++++++++ .../scripts/machines/env.robert_intel | 39 +++++++++++++++ .../scripts/machines/env.underhill_intel | 39 +++++++++++++++ 23 files changed, 761 insertions(+), 105 deletions(-) rename configuration/scripts/machines/{Macros.millikan_intel => Macros.gpsc3_intel} (55%) rename configuration/scripts/machines/{Macros.cesium_intel => Macros.ppp3_intel} (55%) create mode 100644 configuration/scripts/machines/Macros.ppp5_intel create mode 100644 configuration/scripts/machines/Macros.ppp6_gnu create mode 100644 configuration/scripts/machines/Macros.ppp6_gnu-impi create mode 100644 configuration/scripts/machines/Macros.ppp6_intel create mode 100644 configuration/scripts/machines/Macros.ppp6_intel19 create mode 100644 configuration/scripts/machines/Macros.robert_intel create mode 100644 configuration/scripts/machines/Macros.underhill_intel delete mode 100755 configuration/scripts/machines/env.cesium_intel create mode 100644 configuration/scripts/machines/env.gpsc3_intel delete mode 100755 configuration/scripts/machines/env.millikan_intel create mode 100644 configuration/scripts/machines/env.ppp3_intel create mode 100644 configuration/scripts/machines/env.ppp5_intel create mode 100644 configuration/scripts/machines/env.ppp6_gnu create mode 100644 configuration/scripts/machines/env.ppp6_gnu-impi create mode 100644 configuration/scripts/machines/env.ppp6_intel create mode 100644 configuration/scripts/machines/env.ppp6_intel19 create mode 100644 configuration/scripts/machines/env.robert_intel create mode 100644 configuration/scripts/machines/env.underhill_intel diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index 85191dbf6..17ba4bf42 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -234,8 +234,45 @@ cat >> ${jobfile} << EOFB #PBS -j oe #PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds} #PBS -l walltime=${batchtime} +#PBS -W umask=022 EOFB +else if (${ICE_MACHINE} =~ robert* || ${ICE_MACHINE} =~ underhill* || ${ICE_MACHINE} =~ ppp6* || ${ICE_MACHINE} =~ ppp5*) then +cat >> ${jobfile} << EOFB +#PBS -N ${ICE_CASENAME} +#PBS -j oe +#PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds}:mem=20gb +#PBS -l walltime=${batchtime} +#PBS -W umask=022 +EOFB + +else if (${ICE_MACHINE} =~ ppp3*) then +cat >> ${jobfile} << EOFB +#PBS -N ${ICE_CASENAME} +#PBS -j oe +#PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds}:mem=20gb:res_tmpfs=1000:res_image=eccc/eccc_all_ppp_ubuntu-18.04-amd64_latest +#PBS -l walltime=${batchtime} +#PBS -W umask=022 +#PBS -q development +#PBS -o ${ICE_CASEDIR} +#PBS -S /bin/csh +EOFB + +else if (${ICE_MACHINE} =~ gpsc3*) then +cat >> ${jobfile} << EOFB +#SBATCH --export=USER,LOGNAME,HOME,MAIL,PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +#SBATCH -J ${ICE_CASENAME} +#SBATCH -A ${acct} +#SBATCH --partition ${queue} +#SBATCH --time ${batchtime} +#SBATCH --nodes ${nnodes} +#SBATCH --ntasks ${ntasks} +#SBATCH --cpus-per-task ${nthrds} +#SBATCH --mem-per-cpu=5G +#SBATCH --comment="image=eccc/eccc_all_default_ubuntu-18.04-amd64_latest" +EOFB + + else if (${ICE_MACHINE} =~ freya* ) then cat >> ${jobfile} << EOFB #PBS -N ${ICE_CASENAME} diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh index cb9068141..8269e88e2 100755 --- a/configuration/scripts/cice.launch.csh +++ b/configuration/scripts/cice.launch.csh @@ -147,6 +147,43 @@ aprun -n ${ntasks} -N ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_F EOFR endif +#======= +else if (${ICE_MACHINE} =~ ppp5* || ${ICE_MACHINE} =~ ppp6* || ${ICE_MACHINE} =~ robert* || ${ICE_MACHINE} =~ underhill*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +endif + +#======= +else if (${ICE_MACHINE} =~ ppp3*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else +cat >> ${jobfile} << EOFR +rumpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +endif + +#======= +else if (${ICE_MACHINE} =~ gpsc3*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +endif + + #======= else if (${ICE_MACHINE} =~ freya*) then if (${ICE_COMMDIR} =~ serial*) then diff --git a/configuration/scripts/machines/Macros.millikan_intel b/configuration/scripts/machines/Macros.gpsc3_intel similarity index 55% rename from configuration/scripts/machines/Macros.millikan_intel rename to configuration/scripts/machines/Macros.gpsc3_intel index 4a3b21093..916996617 100644 --- a/configuration/scripts/machines/Macros.millikan_intel +++ b/configuration/scripts/machines/Macros.gpsc3_intel @@ -1,5 +1,5 @@ #============================================================================== -# Makefile macros for "millikan" +# Makefile macros for "gpsc3" #============================================================================== # For use with intel compiler #============================================================================== @@ -11,12 +11,12 @@ CFLAGS := -c -O2 -fp-model precise FIXEDFLAGS := -132 FREEFLAGS := -FR -FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -no-wrap-margin +FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -diag-disable 5140 -no-wrap-margin #-xHost -FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays + FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created +#-init=snan,arrays # -heap-arrays 1024 else FFLAGS += -O2 @@ -36,25 +36,14 @@ else endif LD:= $(FC) -NETCDF_PATH := /fs/ssm/hpco/tmp/eccc/201402/04/intel-2016.1.150/ubuntu-14.04-amd64-64/ - -PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs - -#PNETCDF_PATH := $(PNETCDF) -#PNETCDF_PATH := /glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib - -INCLDIR := $(INCLDIR) - -LIB_NETCDF := $(NETCDF_PATH)/lib -LIB_PNETCDF := $(PNETCDF_PATH)/lib -LIB_MPI := $(IMPILIBDIR) - -#SLIBS := -L$(LIB_NETCDF) -lnetcdff -lnetcdf -L$(LIB_PNETCDF) -lpnetcdf -lgptl -SLIBS := -L$(LIB_NETCDF) -lnetcdff -lnetcdf +ifeq ($(ICE_IOTYPE), netcdf) + INCLDIR += $(shell nf-config --fflags) + SLIBS := $(shell nf-config --flibs) +endif ifeq ($(ICE_THREADED), true) - LDFLAGS += -openmp - CFLAGS += -openmp - FFLAGS += -openmp + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp endif diff --git a/configuration/scripts/machines/Macros.cesium_intel b/configuration/scripts/machines/Macros.ppp3_intel similarity index 55% rename from configuration/scripts/machines/Macros.cesium_intel rename to configuration/scripts/machines/Macros.ppp3_intel index 2ad3ff1f3..143445478 100644 --- a/configuration/scripts/machines/Macros.cesium_intel +++ b/configuration/scripts/machines/Macros.ppp3_intel @@ -1,5 +1,5 @@ #============================================================================== -# Makefile macros for "cesium" +# Makefile macros for "ppp3" #============================================================================== # For use with intel compiler #============================================================================== @@ -11,12 +11,12 @@ CFLAGS := -c -O2 -fp-model precise FIXEDFLAGS := -132 FREEFLAGS := -FR -FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -no-wrap-margin +FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -diag-disable 5140 -no-wrap-margin #-xHost -FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays + FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created +#-init=snan,arrays # -heap-arrays 1024 else FFLAGS += -O2 @@ -36,25 +36,14 @@ else endif LD:= $(FC) -NETCDF_PATH := /fs/ssm/hpco/tmp/eccc/201402/04/intel-2016.1.150/ubuntu-14.04-amd64-64/ - -PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs - -#PNETCDF_PATH := $(PNETCDF) -#PNETCDF_PATH := /glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib - -INCLDIR := $(INCLDIR) - -LIB_NETCDF := $(NETCDF_PATH)/lib -LIB_PNETCDF := $(PNETCDF_PATH)/lib -LIB_MPI := $(IMPILIBDIR) - -#SLIBS := -L$(LIB_NETCDF) -lnetcdff -lnetcdf -L$(LIB_PNETCDF) -lpnetcdf -lgptl -SLIBS := -L$(LIB_NETCDF) -lnetcdff -lnetcdf -llapack -lblas +ifeq ($(ICE_IOTYPE), netcdf) + INCLDIR += $(shell nf-config --fflags) + SLIBS := $(shell nf-config --flibs) +endif ifeq ($(ICE_THREADED), true) - LDFLAGS += -openmp - CFLAGS += -openmp - FFLAGS += -openmp + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp endif diff --git a/configuration/scripts/machines/Macros.ppp5_intel b/configuration/scripts/machines/Macros.ppp5_intel new file mode 100644 index 000000000..0a45703b9 --- /dev/null +++ b/configuration/scripts/machines/Macros.ppp5_intel @@ -0,0 +1,49 @@ +#============================================================================== +# Makefile macros for "ppp5" +#============================================================================== +# For use with intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -fp-model precise +#-xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -diag-disable 5140 -no-wrap-margin +#-xHost + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -check nooutput_conversion -debug-parameters all +#-init=snan,arrays +# -heap-arrays 1024 +else + FFLAGS += -O2 +endif + +SCC := icc +SFC := ifort +MPICC := mpiicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +ifeq ($(ICE_IOTYPE), netcdf) + INCLDIR += $(shell nf-config --fflags) + SLIBS := $(shell nf-config --flibs) +endif + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + diff --git a/configuration/scripts/machines/Macros.ppp6_gnu b/configuration/scripts/machines/Macros.ppp6_gnu new file mode 100644 index 000000000..c55406d39 --- /dev/null +++ b/configuration/scripts/machines/Macros.ppp6_gnu @@ -0,0 +1,44 @@ +#============================================================================== +# Makefile macros for ECCC ppp6 +#============================================================================== +# For use with GNU compiler +#============================================================================== + +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 +#-xHost + +FREEFLAGS := -ffree-form +FFLAGS := -fconvert=big-endian -fbacktrace -ffree-line-length-none +#-xHost + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow +else + FFLAGS += -O2 +endif + +SCC := gcc +SFC := gfortran +MPICC := mpicc +MPIFC := mpifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +ifeq ($(ICE_IOTYPE), netcdf) + INCLDIR += $(shell nf-config --fflags) + SLIBS := $(shell nf-config --flibs) +endif + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif diff --git a/configuration/scripts/machines/Macros.ppp6_gnu-impi b/configuration/scripts/machines/Macros.ppp6_gnu-impi new file mode 100644 index 000000000..413154223 --- /dev/null +++ b/configuration/scripts/machines/Macros.ppp6_gnu-impi @@ -0,0 +1,44 @@ +#============================================================================== +# Makefile macros for ECCC ppp6 +#============================================================================== +# For use with GNU compiler +#============================================================================== + +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 +#-xHost + +FREEFLAGS := -ffree-form +FFLAGS := -fconvert=big-endian -fbacktrace -ffree-line-length-none +#-xHost + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow +else + FFLAGS += -O2 +endif + +SCC := gcc +SFC := gfortran +MPICC := mpicc +MPIFC := mpif90 + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +ifeq ($(ICE_IOTYPE), netcdf) + INCLDIR += $(shell nf-config --fflags) + SLIBS := $(shell nf-config --flibs) +endif + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif diff --git a/configuration/scripts/machines/Macros.ppp6_intel b/configuration/scripts/machines/Macros.ppp6_intel new file mode 100644 index 000000000..ece398a49 --- /dev/null +++ b/configuration/scripts/machines/Macros.ppp6_intel @@ -0,0 +1,49 @@ +#============================================================================== +# Makefile macros for "ppp6" +#============================================================================== +# For use with intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -fp-model precise +#-xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -diag-disable 5140 -no-wrap-margin +#-xHost + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -check nooutput_conversion -debug-parameters all +#-init=snan,arrays +# -heap-arrays 1024 +else + FFLAGS += -O2 +endif + +SCC := icc +SFC := ifort +MPICC := mpiicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +ifeq ($(ICE_IOTYPE), netcdf) + INCLDIR += $(shell nf-config --fflags) + SLIBS := $(shell nf-config --flibs) +endif + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + diff --git a/configuration/scripts/machines/Macros.ppp6_intel19 b/configuration/scripts/machines/Macros.ppp6_intel19 new file mode 100644 index 000000000..ece398a49 --- /dev/null +++ b/configuration/scripts/machines/Macros.ppp6_intel19 @@ -0,0 +1,49 @@ +#============================================================================== +# Makefile macros for "ppp6" +#============================================================================== +# For use with intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -fp-model precise +#-xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -diag-disable 5140 -no-wrap-margin +#-xHost + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -check nooutput_conversion -debug-parameters all +#-init=snan,arrays +# -heap-arrays 1024 +else + FFLAGS += -O2 +endif + +SCC := icc +SFC := ifort +MPICC := mpiicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +ifeq ($(ICE_IOTYPE), netcdf) + INCLDIR += $(shell nf-config --fflags) + SLIBS := $(shell nf-config --flibs) +endif + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + diff --git a/configuration/scripts/machines/Macros.robert_intel b/configuration/scripts/machines/Macros.robert_intel new file mode 100644 index 000000000..e54a645c9 --- /dev/null +++ b/configuration/scripts/machines/Macros.robert_intel @@ -0,0 +1,49 @@ +#============================================================================== +# Makefile macros for "robert" +#============================================================================== +# For use with intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -fp-model precise +#-xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -diag-disable 5140 -no-wrap-margin +#-xHost + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -check nooutput_conversion -debug-parameters all +#-init=snan,arrays +# -heap-arrays 1024 +else + FFLAGS += -O2 +endif + +SCC := icc +SFC := ifort +MPICC := mpiicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +ifeq ($(ICE_IOTYPE), netcdf) + INCLDIR += $(shell nf-config --fflags) + SLIBS := $(shell nf-config --flibs) +endif + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + diff --git a/configuration/scripts/machines/Macros.underhill_intel b/configuration/scripts/machines/Macros.underhill_intel new file mode 100644 index 000000000..70a889009 --- /dev/null +++ b/configuration/scripts/machines/Macros.underhill_intel @@ -0,0 +1,49 @@ +#============================================================================== +# Makefile macros for "underhill" +#============================================================================== +# For use with intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -fp-model precise +#-xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -diag-disable 5140 -no-wrap-margin +#-xHost + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -check nooutput_conversion -debug-parameters all +#-init=snan,arrays +# -heap-arrays 1024 +else + FFLAGS += -O2 +endif + +SCC := icc +SFC := ifort +MPICC := mpiicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +ifeq ($(ICE_IOTYPE), netcdf) + INCLDIR += $(shell nf-config --fflags) + SLIBS := $(shell nf-config --flibs) +endif + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + diff --git a/configuration/scripts/machines/env.cesium_intel b/configuration/scripts/machines/env.cesium_intel deleted file mode 100755 index 8dabe1645..000000000 --- a/configuration/scripts/machines/env.cesium_intel +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/csh -f - -set ssmuse=/fs/ssm/main/env/20180430/all/bin/ssmuse-csh # package loader -source $ssmuse -d /fs/ssm/main/opt/intelcomp/intelcomp-2016.1.156 # intel compiler -source /fs/ssm/main/opt/intelcomp/intelcomp-2016.1.156/intelcomp_2016.1.156_multi/bin/compilervars.csh intel64 # should be sourced by above domain, but bug in csh script -source $ssmuse -d /fs/ssm/main/opt/openmpi/openmpi-1.6.5/intelcomp-2016.1.156 # openmpi -source $ssmuse -d /fs/ssm/hpco/tmp/eccc/201402/04/intel-2016.1.150 # netcdf (and openmpi) - -# May be needed for OpenMP memory -#setenv OMP_STACKSIZE 64M - -setenv ICE_MACHINE_MACHNAME cesium -setenv ICE_MACHINE_ENVNAME intel -setenv ICE_MACHINE_MAKE colormake-short -setenv ICE_MACHINE_WKDIR /users/dor/afsg/phb/local/CICEDIRS/CICE_RUNS -setenv ICE_MACHINE_INPUTDATA /users/dor/afsg/phb/local/FORCING -setenv ICE_MACHINE_BASELINE /users/dor/afsg/phb/local/CICEDIRS/CICE_BASELINE -setenv ICE_MACHINE_SUBMIT "qsub" -setenv ICE_MACHINE_TPNODE 36 -setenv ICE_MACHINE_ACCT P0000000 -setenv ICE_MACHINE_QUEUE "debug" -setenv ICE_MACHINE_BLDTHRDS 4 -setenv ICE_MACHINE_QSTAT "qstat " - -if (-e ~/.cice_proj) then - set account_name = `head -1 ~/.cice_proj` - setenv CICE_ACCT ${account_name} -endif diff --git a/configuration/scripts/machines/env.fram_intel b/configuration/scripts/machines/env.fram_intel index 98edb3a66..8b4e97987 100755 --- a/configuration/scripts/machines/env.fram_intel +++ b/configuration/scripts/machines/env.fram_intel @@ -22,8 +22,3 @@ setenv ICE_MACHINE_TPNODE 36 setenv ICE_MACHINE_ACCT P0000000 setenv ICE_MACHINE_BLDTHRDS 1 setenv ICE_MACHINE_QSTAT "qstat " - -if (-e ~/.cice_proj) then - set account_name = `head -1 ~/.cice_proj` - setenv CICE_ACCT ${account_name} -endif diff --git a/configuration/scripts/machines/env.gpsc3_intel b/configuration/scripts/machines/env.gpsc3_intel new file mode 100644 index 000000000..2c8d49275 --- /dev/null +++ b/configuration/scripts/machines/env.gpsc3_intel @@ -0,0 +1,37 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +# SSM package loader +source /fs/ssm/main/opt/ssmuse/ssmuse-1.11/ssmuse_1.11_all/bin/ssmuse-boot.csh >& /dev/null +set ssmuse=`which ssmuse-csh` +# Intel compiler +source $ssmuse -d /fs/ssm/main/intelcomp/intelpsxe-cluster-19.0.3.199 +setenv FOR_DUMP_CORE_FILE 1 +# OpenMPI +source $ssmuse -d /fs/ssm/hpco/exp/openmpi/openmpi-3.1.2--hpcx-2.4.0-mofed-4.6--csh--intel-19.0.3.199 +source $ssmuse -d /fs/ssm/main/opt/openmpi-setup/openmpi-setup-0.3 +# NetCDF +source $ssmuse -d /fs/ssm/hpco/exp/hdf5-netcdf4/serial/static/intel-19.0.3.199/02 + +endif + +setenv ICE_MACHINE_MACHNAME gpsc3 +setenv ICE_MACHINE_ENVNAME intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/data/site3/cice/runs/ +setenv ICE_MACHINE_INPUTDATA /home/ords/cmdd/cmde/sice500/ +setenv ICE_MACHINE_BASELINE ~/data/site3/cice/baseline/ +setenv ICE_MACHINE_SUBMIT "sbatch" +setenv ICE_MACHINE_TPNODE 44 +setenv ICE_MACHINE_ACCT "eccc_cmdd" +setenv ICE_MACHINE_QUEUE "standard" +setenv ICE_MACHINE_BLDTHRDS 24 +setenv ICE_MACHINE_QSTAT "squeue" + +setenv SLURM_EXPORT_ENV "ALL" diff --git a/configuration/scripts/machines/env.millikan_intel b/configuration/scripts/machines/env.millikan_intel deleted file mode 100755 index c0a7356ad..000000000 --- a/configuration/scripts/machines/env.millikan_intel +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/csh -f - -set ssmuse=/fs/ssm/main/env/20180430/all/bin/ssmuse-csh # package loader -source $ssmuse -d /fs/ssm/main/opt/intelcomp/intelcomp-2016.1.156 # intel compiler -source /fs/ssm/main/opt/intelcomp/intelcomp-2016.1.156/intelcomp_2016.1.156_multi/bin/compilervars.csh intel64 # should be sourced by above domain, but bug in csh script -source $ssmuse -d /fs/ssm/main/opt/openmpi/openmpi-1.6.5/intelcomp-2016.1.156 # openmpi -source $ssmuse -d /fs/ssm/hpco/tmp/eccc/201402/04/intel-2016.1.150 # netcdf (and openmpi) - -# May be needed for OpenMP memory -#setenv OMP_STACKSIZE 64M - -setenv ICE_MACHINE_MACHNAME millikan -setenv ICE_MACHINE_ENVNAME intel -setenv ICE_MACHINE_MAKE make -setenv ICE_MACHINE_WKDIR /users/dor/armn/amb/data/local/runs -setenv ICE_MACHINE_INPUTDATA /users/dor/armn/amb/data/local/forcing -setenv ICE_MACHINE_BASELINE /users/dor/armn/amb/data/local/baseline -setenv ICE_MACHINE_SUBMIT "qsub" -setenv ICE_MACHINE_TPNODE 36 -setenv ICE_MACHINE_ACCT P0000000 -setenv ICE_MACHINE_QUEUE "debug" -setenv ICE_MACHINE_BLDTHRDS 4 -setenv ICE_MACHINE_QSTAT "qstat " - -if (-e ~/.cice_proj) then - set account_name = `head -1 ~/.cice_proj` - setenv CICE_ACCT ${account_name} -endif diff --git a/configuration/scripts/machines/env.ppp3_intel b/configuration/scripts/machines/env.ppp3_intel new file mode 100644 index 000000000..cf9fbe4a7 --- /dev/null +++ b/configuration/scripts/machines/env.ppp3_intel @@ -0,0 +1,35 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +# SSM package loader +source /fs/ssm/main/opt/ssmuse/ssmuse-1.11/ssmuse_1.11_all/bin/ssmuse-boot.csh >& /dev/null +set ssmuse=`which ssmuse-csh` +# Intel compiler +source $ssmuse -d comm/eccc/all/opt/intelcomp/intelpsxe-cluster-19.0.3.199 +setenv FOR_DUMP_CORE_FILE 1 +# OpenMPI +source $ssmuse -d /fs/ssm/hpco/exp/openmpi/openmpi-3.1.2--hpcx-2.4.0-mofed-4.6--csh--intel-19.0.3.199 +source $ssmuse -d main/opt/openmpi-setup/openmpi-setup-0.3 +# NetCDF +source $ssmuse -d hpco/exp/hdf5-netcdf4/serial/static/intel-19.0.3.199/02 + +endif + +setenv ICE_MACHINE_MACHNAME ppp3 +setenv ICE_MACHINE_ENVNAME intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/data/site3/cice/runs/ +setenv ICE_MACHINE_INPUTDATA /home/ords/cmdd/cmde/sice500/ +setenv ICE_MACHINE_BASELINE ~/data/site3/cice/baseline/ +setenv ICE_MACHINE_SUBMIT "jobsub" +setenv ICE_MACHINE_TPNODE 36 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "development" +setenv ICE_MACHINE_BLDTHRDS 24 +setenv ICE_MACHINE_QSTAT "jobctl-qstat " diff --git a/configuration/scripts/machines/env.ppp5_intel b/configuration/scripts/machines/env.ppp5_intel new file mode 100644 index 000000000..79dbf2a1b --- /dev/null +++ b/configuration/scripts/machines/env.ppp5_intel @@ -0,0 +1,38 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +# Package loader +source /fs/ssm/main/opt/ssmuse/ssmuse-1.11/ssmuse_1.11_all/bin/ssmuse-boot.csh >& /dev/null +set ssmuse=`which ssmuse-csh` +# Intel compiler + Intel MPI +source $ssmuse -d /fs/ssm/main/opt/intelcomp/inteloneapi-2022.1.2/intelcomp+mpi+mkl +# module use /home/sice500/modulefiles +# module load -s icc mpi +setenv FOR_DUMP_CORE_FILE 1 +setenv I_MPI_DEBUG_COREDUMP 1 +# NetCDF +source $ssmuse -d main/opt/hdf5-netcdf4/serial/shared/inteloneapi-2022.1.2/01 + +# OpenMP +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_MACHNAME ppp5 +setenv ICE_MACHINE_ENVNAME intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/data/ppp5/cice/runs/ +setenv ICE_MACHINE_INPUTDATA /space/hall5/sitestore/eccc/cmd/e/sice500/ +setenv ICE_MACHINE_BASELINE ~/data/ppp5/cice/baselines/ +setenv ICE_MACHINE_SUBMIT qsub +setenv ICE_MACHINE_TPNODE 80 +setenv ICE_MACHINE_ACCT unused +setenv ICE_MACHINE_QUEUE development +setenv ICE_MACHINE_BLDTHRDS 40 +setenv ICE_MACHINE_QSTAT qstat diff --git a/configuration/scripts/machines/env.ppp6_gnu b/configuration/scripts/machines/env.ppp6_gnu new file mode 100644 index 000000000..39cc27740 --- /dev/null +++ b/configuration/scripts/machines/env.ppp6_gnu @@ -0,0 +1,29 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +# OpenMPI +source /usr/mpi/gcc/openmpi-4.1.2a1/bin/mpivars.csh + +# OpenMP +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_MACHNAME ppp6 +setenv ICE_MACHINE_ENVNAME gnu +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/data/site6/cice/runs/ +setenv ICE_MACHINE_INPUTDATA /space/hall6/sitestore/eccc/cmd/e/sice500/ +setenv ICE_MACHINE_BASELINE ~/data/site6/cice/baseline/ +setenv ICE_MACHINE_SUBMIT qsub +setenv ICE_MACHINE_TPNODE 80 +setenv ICE_MACHINE_ACCT unused +setenv ICE_MACHINE_QUEUE development +setenv ICE_MACHINE_BLDTHRDS 40 +setenv ICE_MACHINE_QSTAT qstat diff --git a/configuration/scripts/machines/env.ppp6_gnu-impi b/configuration/scripts/machines/env.ppp6_gnu-impi new file mode 100644 index 000000000..f2a523bf1 --- /dev/null +++ b/configuration/scripts/machines/env.ppp6_gnu-impi @@ -0,0 +1,38 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +# package loader +source /fs/ssm/main/opt/ssmuse/ssmuse-1.11/ssmuse_1.11_all/bin/ssmuse-boot.csh >& /dev/null +set ssmuse=`which ssmuse-csh` +# GNU compilers + Intel MPI +source $ssmuse -x /fs/ssm/main/opt/intelcomp/inteloneapi-2022.1.2 +setenv I_MPI_DEBUG_COREDUMP 1 +setenv I_MPI_F77 gfortran +setenv I_MPI_F90 gfortran +setenv I_MPI_FC gfortran +setenv I_MPI_CC gcc +setenv I_MPI_CXX g++ + +# OpenMP +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_MACHNAME ppp6 +setenv ICE_MACHINE_ENVNAME gnu-impi +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/data/site6/cice/runs/ +setenv ICE_MACHINE_INPUTDATA /space/hall6/sitestore/eccc/cmd/e/sice500/ +setenv ICE_MACHINE_BASELINE ~/data/site6/cice/baseline/ +setenv ICE_MACHINE_SUBMIT qsub +setenv ICE_MACHINE_TPNODE 80 +setenv ICE_MACHINE_ACCT unused +setenv ICE_MACHINE_QUEUE development +setenv ICE_MACHINE_BLDTHRDS 40 +setenv ICE_MACHINE_QSTAT qstat diff --git a/configuration/scripts/machines/env.ppp6_intel b/configuration/scripts/machines/env.ppp6_intel new file mode 100644 index 000000000..dfaeb855f --- /dev/null +++ b/configuration/scripts/machines/env.ppp6_intel @@ -0,0 +1,38 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +# Package loader +source /fs/ssm/main/opt/ssmuse/ssmuse-1.11/ssmuse_1.11_all/bin/ssmuse-boot.csh >& /dev/null +set ssmuse=`which ssmuse-csh` +# Intel compiler + Intel MPI +source $ssmuse -d /fs/ssm/main/opt/intelcomp/inteloneapi-2022.1.2/intelcomp+mpi+mkl +# module use /home/sice500/modulefiles +# module load -s icc mpi +setenv FOR_DUMP_CORE_FILE 1 +setenv I_MPI_DEBUG_COREDUMP 1 +# NetCDF +source $ssmuse -d main/opt/hdf5-netcdf4/serial/shared/inteloneapi-2022.1.2/01 + +# OpenMP +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_MACHNAME ppp6 +setenv ICE_MACHINE_ENVNAME intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/data/ppp6/cice/runs/ +setenv ICE_MACHINE_INPUTDATA /space/hall6/sitestore/eccc/cmd/e/sice500/ +setenv ICE_MACHINE_BASELINE ~/data/ppp6/cice/baselines/ +setenv ICE_MACHINE_SUBMIT qsub +setenv ICE_MACHINE_TPNODE 80 +setenv ICE_MACHINE_ACCT unused +setenv ICE_MACHINE_QUEUE development +setenv ICE_MACHINE_BLDTHRDS 40 +setenv ICE_MACHINE_QSTAT qstat diff --git a/configuration/scripts/machines/env.ppp6_intel19 b/configuration/scripts/machines/env.ppp6_intel19 new file mode 100644 index 000000000..d41242630 --- /dev/null +++ b/configuration/scripts/machines/env.ppp6_intel19 @@ -0,0 +1,39 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +# package loader +source /fs/ssm/main/opt/ssmuse/ssmuse-1.11/ssmuse_1.11_all/bin/ssmuse-boot.csh >& /dev/null +set ssmuse=`which ssmuse-csh` +# Intel compiler +source $ssmuse -d /fs/ssm/main/opt/intelcomp/intelpsxe-cluster-19.0.3.199 +setenv FOR_DUMP_CORE_FILE 1 +# Intel MPI +source $ssmuse -d /fs/ssm/hpco/exp/intelpsxe-impi-19.0.3.199 +setenv FI_PROVIDER verbs +setenv I_MPI_DEBUG_COREDUMP 1 +# NetCDF +source $ssmuse -d hpco/exp/hdf5-netcdf4/serial/static/intel-19.0.3.199/02 + +# OpenMP +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_MACHNAME ppp6 +setenv ICE_MACHINE_ENVNAME intel19 +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/data/site6/cice/runs/ +setenv ICE_MACHINE_INPUTDATA /space/hall6/sitestore/eccc/cmd/e/sice500/ +setenv ICE_MACHINE_BASELINE ~/data/site6/cice/baseline/ +setenv ICE_MACHINE_SUBMIT "qsub" +setenv ICE_MACHINE_TPNODE 80 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "debug" +setenv ICE_MACHINE_BLDTHRDS 24 +setenv ICE_MACHINE_QSTAT "qstat " diff --git a/configuration/scripts/machines/env.robert_intel b/configuration/scripts/machines/env.robert_intel new file mode 100644 index 000000000..43c11a529 --- /dev/null +++ b/configuration/scripts/machines/env.robert_intel @@ -0,0 +1,39 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +# Package loader +source /fs/ssm/main/opt/ssmuse/ssmuse-1.11/ssmuse_1.11_all/bin/ssmuse-boot.csh >& /dev/null +set ssmuse=`which ssmuse-csh` +# Intel compiler + Intel MPI +source $ssmuse -d /fs/ssm/main/opt/intelcomp/inteloneapi-2022.1.2/intelcomp+mpi+mkl +# module use /home/sice500/modulefiles +# module load -s icc mpi +setenv FOR_DUMP_CORE_FILE 1 +setenv I_MPI_DEBUG_COREDUMP 1 +# NetCDF +source $ssmuse -d main/opt/hdf5-netcdf4/serial/shared/inteloneapi-2022.1.2/01 + +# OpenMP +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_MACHNAME robert +setenv ICE_MACHINE_ENVNAME intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/data/robert/cice/runs/ +setenv ICE_MACHINE_INPUTDATA /space/hall6/sitestore/eccc/cmd/e/sice500/ +setenv ICE_MACHINE_BASELINE ~/data/robert/cice/baselines/ +setenv ICE_MACHINE_SUBMIT qsub +setenv ICE_MACHINE_TPNODE 80 +setenv ICE_MACHINE_MAXRUNLENGTH 3 +setenv ICE_MACHINE_ACCT unused +setenv ICE_MACHINE_QUEUE development +setenv ICE_MACHINE_BLDTHRDS 40 +setenv ICE_MACHINE_QSTAT qstat diff --git a/configuration/scripts/machines/env.underhill_intel b/configuration/scripts/machines/env.underhill_intel new file mode 100644 index 000000000..90192853e --- /dev/null +++ b/configuration/scripts/machines/env.underhill_intel @@ -0,0 +1,39 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +# Package loader +source /fs/ssm/main/opt/ssmuse/ssmuse-1.11/ssmuse_1.11_all/bin/ssmuse-boot.csh >& /dev/null +set ssmuse=`which ssmuse-csh` +# Intel compiler + Intel MPI +source $ssmuse -d /fs/ssm/main/opt/intelcomp/inteloneapi-2022.1.2/intelcomp+mpi+mkl +# module use /home/sice500/modulefiles +# module load -s icc mpi +setenv FOR_DUMP_CORE_FILE 1 +setenv I_MPI_DEBUG_COREDUMP 1 +# NetCDF +source $ssmuse -d main/opt/hdf5-netcdf4/serial/shared/inteloneapi-2022.1.2/01 + +# OpenMP +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_MACHNAME underhill +setenv ICE_MACHINE_ENVNAME intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/data/underhill/cice/runs/ +setenv ICE_MACHINE_INPUTDATA /space/hall5/sitestore/eccc/cmd/e/sice500/ +setenv ICE_MACHINE_BASELINE ~/data/underhill/cice/baselines/ +setenv ICE_MACHINE_SUBMIT qsub +setenv ICE_MACHINE_MAXRUNLENGTH 3 +setenv ICE_MACHINE_TPNODE 80 +setenv ICE_MACHINE_ACCT unused +setenv ICE_MACHINE_QUEUE development +setenv ICE_MACHINE_BLDTHRDS 40 +setenv ICE_MACHINE_QSTAT qstat From d41db96b731d0578dd933427838046231c9536e7 Mon Sep 17 00:00:00 2001 From: Philippe Blain Date: Wed, 17 Aug 2022 11:40:35 -0400 Subject: [PATCH 2/6] Update ECCC machine files (#751) * machines: eccc: unify baseline directory * machines: eccc: fix modules initialization Make sure to source the Csh initialization script for environment modules ourselves, as it is not done in all environments. While at it, for convenience add I_MPI_LIBRARY_KIND=debug to the commented lines. (cherry picked from commit 75ef5d260372186845d9beb41795ad930156c2b8) --- configuration/scripts/machines/env.gpsc3_intel | 2 +- configuration/scripts/machines/env.ppp5_intel | 2 ++ configuration/scripts/machines/env.ppp6_gnu | 2 +- configuration/scripts/machines/env.ppp6_gnu-impi | 2 +- configuration/scripts/machines/env.ppp6_intel | 2 ++ configuration/scripts/machines/env.ppp6_intel19 | 2 +- configuration/scripts/machines/env.robert_intel | 2 ++ configuration/scripts/machines/env.underhill_intel | 2 ++ 8 files changed, 12 insertions(+), 4 deletions(-) diff --git a/configuration/scripts/machines/env.gpsc3_intel b/configuration/scripts/machines/env.gpsc3_intel index 2c8d49275..87c7834a4 100644 --- a/configuration/scripts/machines/env.gpsc3_intel +++ b/configuration/scripts/machines/env.gpsc3_intel @@ -26,7 +26,7 @@ setenv ICE_MACHINE_ENVNAME intel setenv ICE_MACHINE_MAKE make setenv ICE_MACHINE_WKDIR ~/data/site3/cice/runs/ setenv ICE_MACHINE_INPUTDATA /home/ords/cmdd/cmde/sice500/ -setenv ICE_MACHINE_BASELINE ~/data/site3/cice/baseline/ +setenv ICE_MACHINE_BASELINE ~/data/site3/cice/baselines/ setenv ICE_MACHINE_SUBMIT "sbatch" setenv ICE_MACHINE_TPNODE 44 setenv ICE_MACHINE_ACCT "eccc_cmdd" diff --git a/configuration/scripts/machines/env.ppp5_intel b/configuration/scripts/machines/env.ppp5_intel index 79dbf2a1b..c4987124a 100644 --- a/configuration/scripts/machines/env.ppp5_intel +++ b/configuration/scripts/machines/env.ppp5_intel @@ -12,7 +12,9 @@ source /fs/ssm/main/opt/ssmuse/ssmuse-1.11/ssmuse_1.11_all/bin/ssmuse-boot.csh > set ssmuse=`which ssmuse-csh` # Intel compiler + Intel MPI source $ssmuse -d /fs/ssm/main/opt/intelcomp/inteloneapi-2022.1.2/intelcomp+mpi+mkl +# source /etc/profile.d/modules.csh # module use /home/sice500/modulefiles +# setenv I_MPI_LIBRARY_KIND debug # module load -s icc mpi setenv FOR_DUMP_CORE_FILE 1 setenv I_MPI_DEBUG_COREDUMP 1 diff --git a/configuration/scripts/machines/env.ppp6_gnu b/configuration/scripts/machines/env.ppp6_gnu index 39cc27740..69ed6ff8b 100644 --- a/configuration/scripts/machines/env.ppp6_gnu +++ b/configuration/scripts/machines/env.ppp6_gnu @@ -20,7 +20,7 @@ setenv ICE_MACHINE_ENVNAME gnu setenv ICE_MACHINE_MAKE make setenv ICE_MACHINE_WKDIR ~/data/site6/cice/runs/ setenv ICE_MACHINE_INPUTDATA /space/hall6/sitestore/eccc/cmd/e/sice500/ -setenv ICE_MACHINE_BASELINE ~/data/site6/cice/baseline/ +setenv ICE_MACHINE_BASELINE ~/data/site6/cice/baselines/ setenv ICE_MACHINE_SUBMIT qsub setenv ICE_MACHINE_TPNODE 80 setenv ICE_MACHINE_ACCT unused diff --git a/configuration/scripts/machines/env.ppp6_gnu-impi b/configuration/scripts/machines/env.ppp6_gnu-impi index f2a523bf1..461e09a43 100644 --- a/configuration/scripts/machines/env.ppp6_gnu-impi +++ b/configuration/scripts/machines/env.ppp6_gnu-impi @@ -29,7 +29,7 @@ setenv ICE_MACHINE_ENVNAME gnu-impi setenv ICE_MACHINE_MAKE make setenv ICE_MACHINE_WKDIR ~/data/site6/cice/runs/ setenv ICE_MACHINE_INPUTDATA /space/hall6/sitestore/eccc/cmd/e/sice500/ -setenv ICE_MACHINE_BASELINE ~/data/site6/cice/baseline/ +setenv ICE_MACHINE_BASELINE ~/data/site6/cice/baselines/ setenv ICE_MACHINE_SUBMIT qsub setenv ICE_MACHINE_TPNODE 80 setenv ICE_MACHINE_ACCT unused diff --git a/configuration/scripts/machines/env.ppp6_intel b/configuration/scripts/machines/env.ppp6_intel index dfaeb855f..ef9396575 100644 --- a/configuration/scripts/machines/env.ppp6_intel +++ b/configuration/scripts/machines/env.ppp6_intel @@ -12,7 +12,9 @@ source /fs/ssm/main/opt/ssmuse/ssmuse-1.11/ssmuse_1.11_all/bin/ssmuse-boot.csh > set ssmuse=`which ssmuse-csh` # Intel compiler + Intel MPI source $ssmuse -d /fs/ssm/main/opt/intelcomp/inteloneapi-2022.1.2/intelcomp+mpi+mkl +# source /etc/profile.d/modules.csh # module use /home/sice500/modulefiles +# setenv I_MPI_LIBRARY_KIND debug # module load -s icc mpi setenv FOR_DUMP_CORE_FILE 1 setenv I_MPI_DEBUG_COREDUMP 1 diff --git a/configuration/scripts/machines/env.ppp6_intel19 b/configuration/scripts/machines/env.ppp6_intel19 index d41242630..6cdd9a036 100644 --- a/configuration/scripts/machines/env.ppp6_intel19 +++ b/configuration/scripts/machines/env.ppp6_intel19 @@ -30,7 +30,7 @@ setenv ICE_MACHINE_ENVNAME intel19 setenv ICE_MACHINE_MAKE make setenv ICE_MACHINE_WKDIR ~/data/site6/cice/runs/ setenv ICE_MACHINE_INPUTDATA /space/hall6/sitestore/eccc/cmd/e/sice500/ -setenv ICE_MACHINE_BASELINE ~/data/site6/cice/baseline/ +setenv ICE_MACHINE_BASELINE ~/data/site6/cice/baselines/ setenv ICE_MACHINE_SUBMIT "qsub" setenv ICE_MACHINE_TPNODE 80 setenv ICE_MACHINE_ACCT P0000000 diff --git a/configuration/scripts/machines/env.robert_intel b/configuration/scripts/machines/env.robert_intel index 43c11a529..d3d9c1eae 100644 --- a/configuration/scripts/machines/env.robert_intel +++ b/configuration/scripts/machines/env.robert_intel @@ -12,7 +12,9 @@ source /fs/ssm/main/opt/ssmuse/ssmuse-1.11/ssmuse_1.11_all/bin/ssmuse-boot.csh > set ssmuse=`which ssmuse-csh` # Intel compiler + Intel MPI source $ssmuse -d /fs/ssm/main/opt/intelcomp/inteloneapi-2022.1.2/intelcomp+mpi+mkl +# source /etc/profile.d/modules.csh # module use /home/sice500/modulefiles +# setenv I_MPI_LIBRARY_KIND debug # module load -s icc mpi setenv FOR_DUMP_CORE_FILE 1 setenv I_MPI_DEBUG_COREDUMP 1 diff --git a/configuration/scripts/machines/env.underhill_intel b/configuration/scripts/machines/env.underhill_intel index 90192853e..bc3eec857 100644 --- a/configuration/scripts/machines/env.underhill_intel +++ b/configuration/scripts/machines/env.underhill_intel @@ -12,7 +12,9 @@ source /fs/ssm/main/opt/ssmuse/ssmuse-1.11/ssmuse_1.11_all/bin/ssmuse-boot.csh > set ssmuse=`which ssmuse-csh` # Intel compiler + Intel MPI source $ssmuse -d /fs/ssm/main/opt/intelcomp/inteloneapi-2022.1.2/intelcomp+mpi+mkl +# source /etc/profile.d/modules.csh # module use /home/sice500/modulefiles +# setenv I_MPI_LIBRARY_KIND debug # module load -s icc mpi setenv FOR_DUMP_CORE_FILE 1 setenv I_MPI_DEBUG_COREDUMP 1 From 801f0c848b20f88b40cfb2c64dde2ec8f59e0583 Mon Sep 17 00:00:00 2001 From: Philippe Blain Date: Thu, 25 Aug 2022 15:26:06 -0400 Subject: [PATCH 3/6] machines: eccc : add ICE_MACHINE_MAXRUNLENGTH to ppp[56] (cherry picked from commit 714afe86f06b61ef172a8fdd28fb0b0764cdd4bc) (this commit is available via git fetch https://github.com/CICE-Consortium/CICE refs/pull/774/head and was squash-merged as part of 16b78da (ice_dyn_vp: allow for bit-for-bit reproducibility under `bfbflag` (#774), 2022-10-20)). --- configuration/scripts/machines/env.ppp5_intel | 1 + configuration/scripts/machines/env.ppp6_gnu | 1 + configuration/scripts/machines/env.ppp6_gnu-impi | 1 + configuration/scripts/machines/env.ppp6_intel | 1 + configuration/scripts/machines/env.ppp6_intel19 | 1 + 5 files changed, 5 insertions(+) diff --git a/configuration/scripts/machines/env.ppp5_intel b/configuration/scripts/machines/env.ppp5_intel index c4987124a..caf566d6b 100644 --- a/configuration/scripts/machines/env.ppp5_intel +++ b/configuration/scripts/machines/env.ppp5_intel @@ -32,6 +32,7 @@ setenv ICE_MACHINE_MAKE make setenv ICE_MACHINE_WKDIR ~/data/ppp5/cice/runs/ setenv ICE_MACHINE_INPUTDATA /space/hall5/sitestore/eccc/cmd/e/sice500/ setenv ICE_MACHINE_BASELINE ~/data/ppp5/cice/baselines/ +setenv ICE_MACHINE_MAXRUNLENGTH 6 setenv ICE_MACHINE_SUBMIT qsub setenv ICE_MACHINE_TPNODE 80 setenv ICE_MACHINE_ACCT unused diff --git a/configuration/scripts/machines/env.ppp6_gnu b/configuration/scripts/machines/env.ppp6_gnu index 69ed6ff8b..cc88c8455 100644 --- a/configuration/scripts/machines/env.ppp6_gnu +++ b/configuration/scripts/machines/env.ppp6_gnu @@ -21,6 +21,7 @@ setenv ICE_MACHINE_MAKE make setenv ICE_MACHINE_WKDIR ~/data/site6/cice/runs/ setenv ICE_MACHINE_INPUTDATA /space/hall6/sitestore/eccc/cmd/e/sice500/ setenv ICE_MACHINE_BASELINE ~/data/site6/cice/baselines/ +setenv ICE_MACHINE_MAXRUNLENGTH 6 setenv ICE_MACHINE_SUBMIT qsub setenv ICE_MACHINE_TPNODE 80 setenv ICE_MACHINE_ACCT unused diff --git a/configuration/scripts/machines/env.ppp6_gnu-impi b/configuration/scripts/machines/env.ppp6_gnu-impi index 461e09a43..2f160f6f4 100644 --- a/configuration/scripts/machines/env.ppp6_gnu-impi +++ b/configuration/scripts/machines/env.ppp6_gnu-impi @@ -30,6 +30,7 @@ setenv ICE_MACHINE_MAKE make setenv ICE_MACHINE_WKDIR ~/data/site6/cice/runs/ setenv ICE_MACHINE_INPUTDATA /space/hall6/sitestore/eccc/cmd/e/sice500/ setenv ICE_MACHINE_BASELINE ~/data/site6/cice/baselines/ +setenv ICE_MACHINE_MAXRUNLENGTH 6 setenv ICE_MACHINE_SUBMIT qsub setenv ICE_MACHINE_TPNODE 80 setenv ICE_MACHINE_ACCT unused diff --git a/configuration/scripts/machines/env.ppp6_intel b/configuration/scripts/machines/env.ppp6_intel index ef9396575..ab3bf737a 100644 --- a/configuration/scripts/machines/env.ppp6_intel +++ b/configuration/scripts/machines/env.ppp6_intel @@ -32,6 +32,7 @@ setenv ICE_MACHINE_MAKE make setenv ICE_MACHINE_WKDIR ~/data/ppp6/cice/runs/ setenv ICE_MACHINE_INPUTDATA /space/hall6/sitestore/eccc/cmd/e/sice500/ setenv ICE_MACHINE_BASELINE ~/data/ppp6/cice/baselines/ +setenv ICE_MACHINE_MAXRUNLENGTH 6 setenv ICE_MACHINE_SUBMIT qsub setenv ICE_MACHINE_TPNODE 80 setenv ICE_MACHINE_ACCT unused diff --git a/configuration/scripts/machines/env.ppp6_intel19 b/configuration/scripts/machines/env.ppp6_intel19 index 6cdd9a036..cfc7623e4 100644 --- a/configuration/scripts/machines/env.ppp6_intel19 +++ b/configuration/scripts/machines/env.ppp6_intel19 @@ -31,6 +31,7 @@ setenv ICE_MACHINE_MAKE make setenv ICE_MACHINE_WKDIR ~/data/site6/cice/runs/ setenv ICE_MACHINE_INPUTDATA /space/hall6/sitestore/eccc/cmd/e/sice500/ setenv ICE_MACHINE_BASELINE ~/data/site6/cice/baselines/ +setenv ICE_MACHINE_MAXRUNLENGTH 6 setenv ICE_MACHINE_SUBMIT "qsub" setenv ICE_MACHINE_TPNODE 80 setenv ICE_MACHINE_ACCT P0000000 From e83081fea18f79ae19dd10f05159bf11d2780c92 Mon Sep 17 00:00:00 2001 From: Philippe Blain Date: Thu, 25 Aug 2022 11:18:15 -0400 Subject: [PATCH 4/6] machines: eccc: use PBS-enabled OpenMPI for 'ppp6_gnu' The system installation of OpenMPI at /usr/mpi/gcc/openmpi-4.1.2a1/ is not compiled with support for PBS. This leads to failures as the MPI runtime does not have the same view of the number of available processors as the job scheduler. Use our own build of OpenMPI, compiled with PBS support, for the 'ppp6_gnu' environment, which uses OpenMPI. (cherry picked from commit cf96fc2ca4d3b89c4dc448268181cd5a7c211862) (this commit is available via git fetch https://github.com/CICE-Consortium/CICE refs/pull/774/head and was squash-merged as part of 16b78da (ice_dyn_vp: allow for bit-for-bit reproducibility under `bfbflag` (#774), 2022-10-20)). --- configuration/scripts/machines/env.ppp6_gnu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/scripts/machines/env.ppp6_gnu b/configuration/scripts/machines/env.ppp6_gnu index cc88c8455..94f478774 100644 --- a/configuration/scripts/machines/env.ppp6_gnu +++ b/configuration/scripts/machines/env.ppp6_gnu @@ -8,7 +8,7 @@ endif if ("$inp" != "-nomodules") then # OpenMPI -source /usr/mpi/gcc/openmpi-4.1.2a1/bin/mpivars.csh +setenv PATH "/home/phb001/.local_rhel-8-icelake-64_gcc/bin:$PATH" # OpenMP setenv OMP_STACKSIZE 64M From ac4fa809232ee7932dbe000befe7406d1d830add Mon Sep 17 00:00:00 2001 From: Philippe Blain Date: Wed, 24 Aug 2022 18:03:40 -0400 Subject: [PATCH 5/6] machines: eccc: set I_MPI_FABRICS=ofi Intel MPI 2021.5.1, which comes with oneAPI 2022.1.2, seems to have an intermittent bug where a call to 'MPI_Waitall' fails with: Abort(17) on node 0 (rank 0 in comm 0): Fatal error in PMPI_Waitall: See the MPI_ERROR field in MPI_Status for the error code and no core dump is produced. This affects at least these cases of the 'decomp' suite: - *_*_restart_gx3_16x2x1x1x800_droundrobin - *_*_restart_gx3_16x2x2x2x200_droundrobin This was reported to Intel and they suggested setting the variable 'I_MPI_FABRICS' to 'ofi' (the default being 'shm:ofi' [1]). This disables shared memory transport and indeeds fixes the failures. Set this variable for all ECCC machine files using Intel MPI. [1] https://www.intel.com/content/www/us/en/develop/documentation/mpi-developer-reference-linux/top/environment-variable-reference/environment-variables-for-fabrics-control/communication-fabrics-control.html (cherry picked from commit 1110dcbc53f9881dbd9fa97188080c08012303f7) (this commit is available via git fetch https://github.com/CICE-Consortium/CICE refs/pull/774/head and was squash-merged as part of 16b78da (ice_dyn_vp: allow for bit-for-bit reproducibility under `bfbflag` (#774), 2022-10-20)). --- configuration/scripts/machines/env.ppp5_intel | 2 ++ configuration/scripts/machines/env.ppp6_gnu-impi | 2 ++ configuration/scripts/machines/env.ppp6_intel | 2 ++ configuration/scripts/machines/env.ppp6_intel19 | 2 ++ configuration/scripts/machines/env.robert_intel | 2 ++ configuration/scripts/machines/env.underhill_intel | 2 ++ 6 files changed, 12 insertions(+) diff --git a/configuration/scripts/machines/env.ppp5_intel b/configuration/scripts/machines/env.ppp5_intel index caf566d6b..6e1a55d5a 100644 --- a/configuration/scripts/machines/env.ppp5_intel +++ b/configuration/scripts/machines/env.ppp5_intel @@ -18,6 +18,8 @@ source $ssmuse -d /fs/ssm/main/opt/intelcomp/inteloneapi-2022.1.2/intelcomp+mpi+ # module load -s icc mpi setenv FOR_DUMP_CORE_FILE 1 setenv I_MPI_DEBUG_COREDUMP 1 +# Stop being buggy +setenv I_MPI_FABRICS ofi # NetCDF source $ssmuse -d main/opt/hdf5-netcdf4/serial/shared/inteloneapi-2022.1.2/01 diff --git a/configuration/scripts/machines/env.ppp6_gnu-impi b/configuration/scripts/machines/env.ppp6_gnu-impi index 2f160f6f4..c5c993b81 100644 --- a/configuration/scripts/machines/env.ppp6_gnu-impi +++ b/configuration/scripts/machines/env.ppp6_gnu-impi @@ -18,6 +18,8 @@ setenv I_MPI_F90 gfortran setenv I_MPI_FC gfortran setenv I_MPI_CC gcc setenv I_MPI_CXX g++ +# Stop being buggy +setenv I_MPI_FABRICS ofi # OpenMP setenv OMP_STACKSIZE 64M diff --git a/configuration/scripts/machines/env.ppp6_intel b/configuration/scripts/machines/env.ppp6_intel index ab3bf737a..cfb05e83a 100644 --- a/configuration/scripts/machines/env.ppp6_intel +++ b/configuration/scripts/machines/env.ppp6_intel @@ -18,6 +18,8 @@ source $ssmuse -d /fs/ssm/main/opt/intelcomp/inteloneapi-2022.1.2/intelcomp+mpi+ # module load -s icc mpi setenv FOR_DUMP_CORE_FILE 1 setenv I_MPI_DEBUG_COREDUMP 1 +# Stop being buggy +setenv I_MPI_FABRICS ofi # NetCDF source $ssmuse -d main/opt/hdf5-netcdf4/serial/shared/inteloneapi-2022.1.2/01 diff --git a/configuration/scripts/machines/env.ppp6_intel19 b/configuration/scripts/machines/env.ppp6_intel19 index cfc7623e4..a8421af0f 100644 --- a/configuration/scripts/machines/env.ppp6_intel19 +++ b/configuration/scripts/machines/env.ppp6_intel19 @@ -17,6 +17,8 @@ setenv FOR_DUMP_CORE_FILE 1 source $ssmuse -d /fs/ssm/hpco/exp/intelpsxe-impi-19.0.3.199 setenv FI_PROVIDER verbs setenv I_MPI_DEBUG_COREDUMP 1 +# Stop being buggy +setenv I_MPI_FABRICS ofi # NetCDF source $ssmuse -d hpco/exp/hdf5-netcdf4/serial/static/intel-19.0.3.199/02 diff --git a/configuration/scripts/machines/env.robert_intel b/configuration/scripts/machines/env.robert_intel index d3d9c1eae..592d765fe 100644 --- a/configuration/scripts/machines/env.robert_intel +++ b/configuration/scripts/machines/env.robert_intel @@ -18,6 +18,8 @@ source $ssmuse -d /fs/ssm/main/opt/intelcomp/inteloneapi-2022.1.2/intelcomp+mpi+ # module load -s icc mpi setenv FOR_DUMP_CORE_FILE 1 setenv I_MPI_DEBUG_COREDUMP 1 +# Stop being buggy +setenv I_MPI_FABRICS ofi # NetCDF source $ssmuse -d main/opt/hdf5-netcdf4/serial/shared/inteloneapi-2022.1.2/01 diff --git a/configuration/scripts/machines/env.underhill_intel b/configuration/scripts/machines/env.underhill_intel index bc3eec857..dee0ab92b 100644 --- a/configuration/scripts/machines/env.underhill_intel +++ b/configuration/scripts/machines/env.underhill_intel @@ -18,6 +18,8 @@ source $ssmuse -d /fs/ssm/main/opt/intelcomp/inteloneapi-2022.1.2/intelcomp+mpi+ # module load -s icc mpi setenv FOR_DUMP_CORE_FILE 1 setenv I_MPI_DEBUG_COREDUMP 1 +# Stop being buggy +setenv I_MPI_FABRICS ofi # NetCDF source $ssmuse -d main/opt/hdf5-netcdf4/serial/shared/inteloneapi-2022.1.2/01 From 739d81585435ac1cbeb22292d62a0122d8d2c3f9 Mon Sep 17 00:00:00 2001 From: Philippe Blain Date: Wed, 24 Aug 2022 18:19:31 -0400 Subject: [PATCH 6/6] machines: eccc: set I_MPI_CBWR for BASEGEN/BASECOM runs Intel MPI, in contrast to OpenMPI (as far as I was able to test, and see [1], [2]), does not (by default) guarantee that repeated runs of the same code on the same machine with the same number of MPI ranks yield the same results when collective operations (e.g. 'MPI_ALLREDUCE') are used. Since the VP solver uses MPI_ALLREDUCE in its algorithm, this leads to repeated runs of the code giving different answers, and baseline comparing runs with code built from the same commit failing. When generating a baseline or comparing against an existing baseline, set the environment variable 'I_MPI_CBWR' to 1 for ECCC machine files using Intel MPI [3], so that (processor) topology-aware collective algorithms are not used and results are reproducible. Note that we do not need to set this variable on robert or underhill, on which jobs have exclusive node access and thus job placement (on processors) is guaranteed to be reproducible. [1] https://stackoverflow.com/a/45916859/ [2] https://scicomp.stackexchange.com/a/2386/ [3] https://www.intel.com/content/www/us/en/develop/documentation/mpi-developer-reference-linux/top/environment-variable-reference/i-mpi-adjust-family-environment-variables.html#i-mpi-adjust-family-environment-variables_GUID-A5119508-5588-4CF5-9979-8D60831D1411 (cherry picked from commit 1f46305ada618fc8dff9eacfbbde7ab26a27fd10) (this commit is available via git fetch https://github.com/CICE-Consortium/CICE refs/pull/774/head and was squash-merged as part of 16b78da (ice_dyn_vp: allow for bit-for-bit reproducibility under `bfbflag` (#774), 2022-10-20)). --- configuration/scripts/machines/env.ppp5_intel | 4 ++++ configuration/scripts/machines/env.ppp6_gnu-impi | 4 ++++ configuration/scripts/machines/env.ppp6_intel | 4 ++++ configuration/scripts/machines/env.ppp6_intel19 | 4 ++++ 4 files changed, 16 insertions(+) diff --git a/configuration/scripts/machines/env.ppp5_intel b/configuration/scripts/machines/env.ppp5_intel index 6e1a55d5a..c6e272692 100644 --- a/configuration/scripts/machines/env.ppp5_intel +++ b/configuration/scripts/machines/env.ppp5_intel @@ -18,6 +18,10 @@ source $ssmuse -d /fs/ssm/main/opt/intelcomp/inteloneapi-2022.1.2/intelcomp+mpi+ # module load -s icc mpi setenv FOR_DUMP_CORE_FILE 1 setenv I_MPI_DEBUG_COREDUMP 1 +# Reproducible collectives +if (${ICE_BASEGEN} != ${ICE_SPVAL} || ${ICE_BASECOM} != ${ICE_SPVAL}) then + setenv I_MPI_CBWR 1 +endif # Stop being buggy setenv I_MPI_FABRICS ofi # NetCDF diff --git a/configuration/scripts/machines/env.ppp6_gnu-impi b/configuration/scripts/machines/env.ppp6_gnu-impi index c5c993b81..61cd46ed9 100644 --- a/configuration/scripts/machines/env.ppp6_gnu-impi +++ b/configuration/scripts/machines/env.ppp6_gnu-impi @@ -18,6 +18,10 @@ setenv I_MPI_F90 gfortran setenv I_MPI_FC gfortran setenv I_MPI_CC gcc setenv I_MPI_CXX g++ +# Reproducible collectives +if (${ICE_BASEGEN} != ${ICE_SPVAL} || ${ICE_BASECOM} != ${ICE_SPVAL}) then + setenv I_MPI_CBWR 1 +endif # Stop being buggy setenv I_MPI_FABRICS ofi diff --git a/configuration/scripts/machines/env.ppp6_intel b/configuration/scripts/machines/env.ppp6_intel index cfb05e83a..faecb9b5a 100644 --- a/configuration/scripts/machines/env.ppp6_intel +++ b/configuration/scripts/machines/env.ppp6_intel @@ -18,6 +18,10 @@ source $ssmuse -d /fs/ssm/main/opt/intelcomp/inteloneapi-2022.1.2/intelcomp+mpi+ # module load -s icc mpi setenv FOR_DUMP_CORE_FILE 1 setenv I_MPI_DEBUG_COREDUMP 1 +# Reproducible collectives +if (${ICE_BASEGEN} != ${ICE_SPVAL} || ${ICE_BASECOM} != ${ICE_SPVAL}) then + setenv I_MPI_CBWR 1 +endif # Stop being buggy setenv I_MPI_FABRICS ofi # NetCDF diff --git a/configuration/scripts/machines/env.ppp6_intel19 b/configuration/scripts/machines/env.ppp6_intel19 index a8421af0f..554b31e42 100644 --- a/configuration/scripts/machines/env.ppp6_intel19 +++ b/configuration/scripts/machines/env.ppp6_intel19 @@ -17,6 +17,10 @@ setenv FOR_DUMP_CORE_FILE 1 source $ssmuse -d /fs/ssm/hpco/exp/intelpsxe-impi-19.0.3.199 setenv FI_PROVIDER verbs setenv I_MPI_DEBUG_COREDUMP 1 +# Reproducible collectives +if (${ICE_BASEGEN} != ${ICE_SPVAL} || ${ICE_BASECOM} != ${ICE_SPVAL}) then + setenv I_MPI_CBWR 1 +endif # Stop being buggy setenv I_MPI_FABRICS ofi # NetCDF