Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Port the GDAS_INIT utility to Jet #563

Merged
merged 6 commits into from
Aug 13, 2021
Merged

Port the GDAS_INIT utility to Jet #563

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
0227a9c
Baseline some initial commits to run the gdas_init utility
GeorgeGayno-NOAA Aug 9, 2021
17bbfde
Update to run chgres for v16 data.
GeorgeGayno-NOAA Aug 10, 2021
cc428ff
Update script to run the chgres step for gfs v14 on Jet.
GeorgeGayno-NOAA Aug 10, 2021
d600f35
Update script to run remaining chgres cases on Jet.
GeorgeGayno-NOAA Aug 11, 2021
83bad46
Merge branch 'develop' into feature/gdas_init_jet
GeorgeGayno-NOAA Aug 11, 2021
21a0cd5
Merge branch 'develop' into feature/gdas_init_jet
GeorgeGayno-NOAA Aug 13, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions modulefiles/build.jet.intel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
## Build and run module for Jet
#############################################################
module load cmake/3.16.1
module load hpss

module use /lfs4/HFIP/hfv3gfs/nwprod/hpc-stack/libs/modulefiles/stack
module load hpc/1.1.0
Expand All @@ -23,3 +24,4 @@ module load sfcio/1.4.1
module load nemsio/2.5.2
module load g2/3.4.1
module load wgrib2/2.0.8
module load prod_util/1.2.2
8 changes: 4 additions & 4 deletions util/gdas_init/config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,17 +47,17 @@
#
#-----------------------------------------------------------

EXTRACT_DIR=/gpfs/dell1/stmp/$USER/gdas.init/input
EXTRACT_DIR=/lfs4/HFIP/emcda/$USER/stmp/gdas.init/input
EXTRACT_DATA=yes

RUN_CHGRES=yes
RUN_CHGRES=no

yy=2021
mm=03
dd=21
hh=06

use_v16retro=no
use_v16retro=yes

LEVS=65

Expand All @@ -68,7 +68,7 @@ CRES_ENKF=C96

UFS_DIR=$PWD/../..

OUTDIR=/gpfs/dell1/stmp/$USER/gdas.init/output
OUTDIR=/lfs4/HFIP/emcda/$USER/stmp/gdas.init/output

#---------------------------------------------------------
# Dont touch anything below here.
Expand Down
212 changes: 212 additions & 0 deletions util/gdas_init/driver.jet.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,212 @@
#!/bin/bash

#---------------------------------------------------------------------
# Driver script for running on Jet.
#
# Edit the 'config' file before running.
#
# Adjust the project code and queue as necessary.
#---------------------------------------------------------------------

set -x

compiler=${compiler:-"intel"}
source ../../sorc/machine-setup.sh > /dev/null 2>&1
module use ../../modulefiles
module load build.$target.$compiler
module list

PROJECT_CODE=hfv3gfs
QUEUE=batch
PARTITION=xjet

source config

if [ $EXTRACT_DATA == yes ]; then

rm -fr $EXTRACT_DIR
mkdir -p $EXTRACT_DIR

MEM=6000M
WALLT="2:00:00"

case $gfs_ver in
v12 | v13)
DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \
-o log.data.${CDUMP} -e log.data.${CDUMP} ./get_pre-v14.data.sh ${CDUMP})
DEPEND="-d afterok:$DATAH"
if [ "$CDUMP" = "gdas" ] ; then
DATA1=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_enkf \
-o log.data.enkf -e log.data.enkf ./get_pre-v14.data.sh enkf)
DEPEND="-d afterok:$DATAH:$DATA1"
fi
;;
v14)
DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \
-o log.data.${CDUMP} -e log.data.${CDUMP} ./get_v14.data.sh ${CDUMP})
DEPEND="-d afterok:$DATAH"
if [ "$CDUMP" = "gdas" ] ; then
DATA1=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_enkf \
-o log.data.enkf -e log.data.enkf ./get_v14.data.sh enkf)
DEPEND="-d afterok:$DATAH:$DATA1"
fi
;;
v15)
if [ "$CDUMP" = "gfs" ] ; then
DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \
-o log.data.${CDUMP} -e log.data.${CDUMP} ./get_v15.data.sh ${CDUMP})
DEPEND="-d afterok:$DATAH"
else
DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \
-o log.data.${CDUMP} -e log.data.${CDUMP} ./get_v15.data.sh ${CDUMP})
DATA1=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp1 \
-o log.data.grp1 -e log.data.grp1 ./get_v15.data.sh grp1)
DATA2=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp2 \
-o log.data.grp2 -e log.data.grp2 ./get_v15.data.sh grp2)
DATA3=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp3 \
-o log.data.grp3 -e log.data.grp3 ./get_v15.data.sh grp3)
DATA4=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp4 \
-o log.data.grp4 -e log.data.grp4 ./get_v15.data.sh grp4)
DATA5=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp5 \
-o log.data.grp5 -e log.data.grp5 ./get_v15.data.sh grp5)
DATA6=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp6 \
-o log.data.grp6 -e log.data.grp6 ./get_v15.data.sh grp6)
DATA7=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp7 \
-o log.data.grp7 -e log.data.grp7 ./get_v15.data.sh grp7)
DATA8=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp8 \
-o log.data.grp8 -e log.data.grp8 ./get_v15.data.sh grp8)
DEPEND="-d afterok:$DATAH:$DATA1:$DATA2:$DATA3:$DATA4:$DATA5:$DATA6:$DATA7:$DATA8"
fi
;;
v16retro)
DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_v16retro \
-o log.data.v16retro -e log.data.v16retro ./get_v16retro.data.sh ${CDUMP})
DEPEND="-d afterok:$DATAH"
;;
v16)
DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \
-o log.data.${CDUMP} -e log.data.${CDUMP} ./get_v16.data.sh ${CDUMP})
DEPEND="-d afterok:$DATAH"
if [ "$CDUMP" = "gdas" ] ; then
DATA1=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp1 \
-o log.data.grp1 -e log.data.grp1 ./get_v16.data.sh grp1)
DATA2=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp2 \
-o log.data.grp2 -e log.data.grp2 ./get_v16.data.sh grp2)
DATA3=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp3 \
-o log.data.grp3 -e log.data.grp3 ./get_v16.data.sh grp3)
DATA4=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp4 \
-o log.data.grp4 -e log.data.grp4 ./get_v16.data.sh grp4)
DATA5=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp5 \
-o log.data.grp5 -e log.data.grp5 ./get_v16.data.sh grp5)
DATA6=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp6 \
-o log.data.grp6 -e log.data.grp6 ./get_v16.data.sh grp6)
DATA7=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp7 \
-o log.data.grp7 -e log.data.grp7 ./get_v16.data.sh grp7)
DATA8=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp8 \
-o log.data.grp8 -e log.data.grp8 ./get_v16.data.sh grp8)
DEPEND="-d afterok:$DATAH:$DATA1:$DATA2:$DATA3:$DATA4:$DATA5:$DATA6:$DATA7:$DATA8"
fi
;;
esac

else # do not extract data.

DEPEND=' '

fi # extract data?

if [ $RUN_CHGRES == yes ]; then

export APRUN=srun
NODES=3
WALLT="0:15:00"
export OMP_NUM_THREADS=1
if [ $CRES_HIRES == 'C768' ] ; then
NODES=5
elif [ $CRES_HIRES == 'C1152' ] ; then
NODES=8
WALLT="0:20:00"
fi
case $gfs_ver in
v12 | v13)
export OMP_NUM_THREADS=4
export OMP_STACKSIZE=1024M
sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} --cpus-per-task=$OMP_NUM_THREADS \
-t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \
-o log.${CDUMP} -e log.${CDUMP} --partition=${PARTITION} ${DEPEND} run_pre-v14.chgres.sh ${CDUMP}
;;
v14)
sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \
-o log.${CDUMP} -e log.${CDUMP} --partition=${PARTITION} ${DEPEND} run_v14.chgres.sh ${CDUMP}
;;
v15)
if [ "$CDUMP" = "gdas" ]; then
sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \
-o log.${CDUMP} -e log.${CDUMP} --partition=${PARTITION} ${DEPEND} run_v15.chgres.sh ${CDUMP}
else
sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \
-o log.${CDUMP} -e log.${CDUMP} --partition=${PARTITION} ${DEPEND} run_v15.chgres.gfs.sh
fi
;;
v16retro)
if [ "$CDUMP" = "gdas" ] ; then
sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \
-o log.${CDUMP} -e log.${CDUMP} --partition=${PARTITION} ${DEPEND} run_v16retro.chgres.sh hires
else
sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \
-o log.${CDUMP} -e log.${CDUMP} --partition=${PARTITION} ${DEPEND} run_v16.chgres.sh ${CDUMP}
fi
;;
v16)
sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \
-o log.${CDUMP} -e log.${CDUMP} --partition=${PARTITION} ${DEPEND} run_v16.chgres.sh ${CDUMP}
;;
esac

if [ "$CDUMP" = "gdas" ]; then

WALLT="0:15:00"

if [ "$gfs_ver" = "v16retro" ]; then

sbatch --parsable --ntasks-per-node=12 --nodes=1 -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_enkf \
-o log.enkf -e log.enkf --partition=${PARTITION} ${DEPEND} run_v16retro.chgres.sh enkf

else

MEMBER=1
while [ $MEMBER -le 80 ]; do
if [ $MEMBER -lt 10 ]; then
MEMBER_CH="00${MEMBER}"
else
MEMBER_CH="0${MEMBER}"
fi
case $gfs_ver in
v12 | v13)
export OMP_NUM_THREADS=2
export OMP_STACKSIZE=1024M
sbatch --parsable --ntasks-per-node=12 --nodes=1 --cpus-per-task=$OMP_NUM_THREADS \
-t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \
-o log.${MEMBER_CH} -e log.${MEMBER_CH} --partition=${PARTITION} ${DEPEND} run_pre-v14.chgres.sh ${MEMBER_CH}
;;
v14)
sbatch --parsable --ntasks-per-node=12 --nodes=1 -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \
-o log.${MEMBER_CH} -e log.${MEMBER_CH} --partition=${PARTITION} ${DEPEND} run_v14.chgres.sh ${MEMBER_CH}
;;
v15)
sbatch --parsable --ntasks-per-node=12 --nodes=1 -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \
-o log.${MEMBER_CH} -e log.${MEMBER_CH} --partition=${PARTITION} ${DEPEND} run_v15.chgres.sh ${MEMBER_CH}
;;
v16)
sbatch --parsable --ntasks-per-node=12 --nodes=1 -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \
-o log.${MEMBER_CH} -e log.${MEMBER_CH} --partition=${PARTITION} ${DEPEND} run_v16.chgres.sh ${MEMBER_CH}
;;
esac
MEMBER=$(( $MEMBER + 1 ))
done

fi # Is this v16 retro?

fi # which CDUMP?

fi # run chgres?