From 138adab11cf247896faeaed48f5c3dd6c2218c85 Mon Sep 17 00:00:00 2001
From: Anton Steketee <79179784+anton-seaice@users.noreply.github.com>
Date: Mon, 18 Dec 2023 14:39:54 +1100
Subject: [PATCH] Standalone gadi setup (#1)

---
 configuration/scripts/cice.batch.csh          | 25 +++++++
 configuration/scripts/cice.launch.csh         | 12 ++++
 .../scripts/machines/Macros.gadi_intel        | 71 +++++++++++++++++++
 configuration/scripts/machines/env.gadi_intel | 56 +++++++++++++++
 4 files changed, 164 insertions(+)
 create mode 100644 configuration/scripts/machines/Macros.gadi_intel
 create mode 100644 configuration/scripts/machines/env.gadi_intel

diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh
index 33b27cbf8..b428a1ffd 100755
--- a/configuration/scripts/cice.batch.csh
+++ b/configuration/scripts/cice.batch.csh
@@ -48,6 +48,31 @@ cat >> ${jobfile} << EOFB
 ###PBS -m be
 EOFB
 
+else if (${ICE_MACHINE} =~ gadi*) then
+if (${queue} =~ *sr) then #sapphire rapids
+  @ memuse = ( $ncores * 481 / 100 )
+else if (${queue} =~ *bw) then #broadwell
+  @ memuse = ( $ncores * 457 / 100 )
+else if (${queue} =~ *sl) then 
+  @ memuse = ( $ncores * 6 )
+else #normal queues
+  @ memuse = ( $ncores * 395 / 100 )
+endif
+cat >> ${jobfile} << EOFB
+#PBS -q ${queue}
+#PBS -P ${ICE_MACHINE_PROJ}
+#PBS -N ${ICE_CASENAME}
+#PBS -l storage=gdata/${ICE_MACHINE_PROJ}+scratch/${ICE_MACHINE_PROJ}+gdata/ik11
+#PBS -l ncpus=${ncores}
+#PBS -l mem=${memuse}gb
+#PBS -l walltime=${batchtime}
+#PBS -j oe 
+#PBS -W umask=003
+#PBS -o ${ICE_CASEDIR}
+source /etc/profile.d/modules.csh
+module use `echo ${MODULEPATH} | sed 's/:/ /g'` #copy the users modules
+EOFB
+
 else if (${ICE_MACHINE} =~ gust*) then
 cat >> ${jobfile} << EOFB
 #PBS -q ${queue}
diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh
index f8347e101..f8a177b46 100755
--- a/configuration/scripts/cice.launch.csh
+++ b/configuration/scripts/cice.launch.csh
@@ -46,6 +46,18 @@ mpiexec --cpu-bind depth -n ${ntasks} -ppn ${taskpernodelimit} -d ${nthrds} ./ci
 EOFR
 endif
 
+#=======
+else if (${ICE_MACHCOMP} =~ gadi*) then
+if (${ICE_COMMDIR} =~ serial*) then
+cat >> ${jobfile} << EOFR
+./cice >&! \$ICE_RUNLOG_FILE
+EOFR
+else
+cat >> ${jobfile} << EOFR
+mpirun -n ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE
+EOFR
+endif
+
 #=======
 else if (${ICE_MACHCOMP} =~ hobart* || ${ICE_MACHCOMP} =~ izumi*) then
 if (${ICE_COMMDIR} =~ serial*) then
diff --git a/configuration/scripts/machines/Macros.gadi_intel b/configuration/scripts/machines/Macros.gadi_intel
new file mode 100644
index 000000000..152566490
--- /dev/null
+++ b/configuration/scripts/machines/Macros.gadi_intel
@@ -0,0 +1,71 @@
+#==============================================================================
+# Makefile macros for NCI Gadi, intel compiler
+#==============================================================================
+
+CPP        := fpp
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c -O2 -fp-model precise
+
+FIXEDFLAGS := -132
+FREEFLAGS  := -FR
+FFLAGS     := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -link_mpi=dbg -stand f08
+else
+  FFLAGS     += -O2
+endif
+
+SCC   := icx 
+SFC   := ifort
+MPICC := mpicc
+MPIFC := mpifort
+
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+LD:= $(FC)
+
+SLIBS := $(SLIBS)
+INCLDIR := $(INCLDIR)
+
+ifndef SPACK_NETCDF_FORTRAN_ROOT  
+   SLIBS += -L$(NETCDF)/lib -lnetcdf -lnetcdff
+else
+   SLIBS += -L$(SPACK_NETCDF_C_ROOT)/lib64 -lnetcdf
+   SLIBS += -L$(SPACK_NETCDF_FORTRAN_ROOT)/lib  -lnetcdff
+   INCLDIR += -I $(SPACK_NETCDF_C_ROOT)/include
+   INCLDIR += -I $(SPACK_NETCDF_FORTRAN_ROOT)/include
+endif
+
+#LIB_PNETCDF := $(PNETCDF)/lib
+#SLIBS   += L$(LIB_PNETCDF) -lpnetcdf
+
+ifeq ($(ICE_THREADED), true) 
+   LDFLAGS += -qopenmp 
+   CFLAGS += -qopenmp 
+   FFLAGS += -qopenmp 
+endif
+
+ifeq ($(ICE_IOTYPE), pio1)
+   LIB_PIO := $(PIO_LIBDIR)
+   SLIBS   += -L$(LIB_PIO) -lpio
+endif
+
+ifeq ($(ICE_IOTYPE), pio2)
+  ifndef SPACK_PARALLELIO_ROOT
+    SLIBS += -L$(PARALLELIO_ROOT)/lib -lpioc -lpiof
+  else
+   SLIBS += -L$(SPACK_PARALLELIO_ROOT)/lib -lpioc -lpiof  
+   INCLDIR += -I $(SPACK_PARALLELIO_ROOT)/include
+  endif
+   
+  SLIBS += $(SLIBS) -L$(OMPI_BASE)/lib -lmpi_usempif08 -lmpi_usempi_ignore_tkr -lmpi_mpifh 
+   
+endif
+
diff --git a/configuration/scripts/machines/env.gadi_intel b/configuration/scripts/machines/env.gadi_intel
new file mode 100644
index 000000000..ece74fc72
--- /dev/null
+++ b/configuration/scripts/machines/env.gadi_intel
@@ -0,0 +1,56 @@
+#!/bin/csh -f
+
+set inp = "undefined"
+if ($#argv == 1) then
+  set inp = $1
+endif
+
+if ("$inp" != "-nomodules") then
+
+  source /etc/profile.d/modules.csh
+
+  module load intel-compiler
+  module load openmpi
+  
+  if ($?ICE_IOTYPE) then
+    if ($ICE_IOTYPE =~ pio*) then
+      if ($ICE_IOTYPE == "pio1") then
+        # we don't have pio1 installed anywhere  
+        module load pnetcdf
+        module load netcdf
+        module load pio
+      else
+        module load parallelio
+      endif
+    else 
+      module load netcdf
+    endif
+  endif
+
+  if ($?ICE_BFBTYPE) then
+    if ($ICE_BFBTYPE =~ qcchk*) then
+      # conda/analysis has the required librarys, skip building from cice yaml file
+      module use /g/data/hh5/public/modules
+      module load conda/analysis
+      # conda env create -f ../../configuration/scripts/tests/qctest.yml
+      # conda activate qctest
+    endif
+  endif
+
+endif
+
+setenv ICE_MACHINE_MACHNAME gadi
+setenv ICE_MACHINE_MACHINFO "Intel Xeon Scalable"
+setenv ICE_MACHINE_ENVNAME intel
+# setenv ICE_MACHINE_ENVINFO ${module list}
+setenv ICE_MACHINE_MAKE gmake
+setenv ICE_MACHINE_WKDIR /scratch/$PROJECT/$USER/CICE_RUNS
+setenv ICE_MACHINE_INPUTDATA /g/data/ik11/inputs
+setenv ICE_MACHINE_BASELINE /scratch/$PROJECT/$USER/CICE_BASELINE
+setenv ICE_MACHINE_SUBMIT "qsub"
+setenv ICE_MACHINE_PROJ $PROJECT
+setenv ICE_MACHINE_ACCT $USER
+setenv ICE_MACHINE_QUEUE "normal"
+setenv ICE_MACHINE_TPNODE 48
+setenv ICE_MACHINE_BLDTHRDS 4
+setenv ICE_MACHINE_QSTAT "qstat"