Merge pull request #132 from hafs-community/feature/hafsv0.3_phase2

HAFSv0.3 moving-nesting related coupling, vortex initialization, and data assimilation capabilities
hafs-community · Apr 29, 2022 · 41b70b8 · 41b70b8
2 parents 6002749 + 7e56220
commit 41b70b8
Show file tree

Hide file tree

Showing 217 changed files with 33,567 additions and 5,472 deletions.
diff --git a/jobs/JHAFS_ANALYSIS b/jobs/JHAFS_ANALYSIS
@@ -39,6 +39,11 @@ export cyc=${cyc:?}
 export STORM=${STORM:-FAKE}
 export STORMID=${STORMID:-00L}
 export ENSDA=${ENSDA:-NO}
+export GSI_D01=${GSI_D01:-NO}
+export GSI_D02=${GSI_D02:-NO}
+
+# analysis for d01
+if [ ${GSI_D01} = YES ]; then
 
 export DATA=${WORKhafs}/analysis
 
@@ -52,12 +57,38 @@ cd $DATA
 
 # Execute ex-script
 ${HOMEhafs}/scripts/exhafs_analysis.sh
-export err=$?
-exit $err
 
 export KEEPDATA=${KEEPDATA:-YES}
 if [ "${KEEPDATA^^}" != YES ]; then
   rm -rf $DATA
 fi
 
+fi
+
+# analysis for d02
+if [ ${GSI_D02} = YES ]; then
+
+export neststr=".nest02"
+export tilestr=".tile2"
+export nesttilestr=".nest02.tile2"
+export DATA=${WORKhafs}/analysis_d02
+
+export SCRUBDATA=${SCRUBDATA:-YES}
+if [ "${SCRUBDATA}" = YES ]; then
+  rm -rf $DATA
+fi
+
+mkdir -p $DATA
+cd $DATA
+
+# Execute ex-script
+${HOMEhafs}/scripts/exhafs_analysis.sh
+
+export KEEPDATA=${KEEPDATA:-YES}
+if [ "${KEEPDATA^^}" != YES ]; then
+  rm -rf $DATA
+fi
+
+fi
+
 date
diff --git a/jobs/JHAFS_ANALYSIS_VR b/jobs/JHAFS_ANALYSIS_VR
@@ -62,8 +62,6 @@ cd $DATA
 
 # Execute ex-script
 ${HOMEhafs}/scripts/exhafs_analysis_vr.sh
-export err=$?
-exit $err
 
 export KEEPDATA=${KEEPDATA:-YES}
 if [ "${KEEPDATA^^}" != YES ]; then

diff --git a/jobs/JHAFS_ATM_IC b/jobs/JHAFS_ATM_IC
@@ -41,15 +41,20 @@ export cyc=${cyc:?}
 export STORM=${STORM:-FAKE}
 export STORMID=${STORMID:-00L}
 export ENSDA=${ENSDA:-NO}
+export FGAT_MODEL=${FGAT_MODEL:-gfs}
+export FGAT_HR=${FGAT_HR:-00}
 
 # Deterministic or ensemble
-if [ ${ENSDA} != YES ]; then
-  export OUTDIR=${OUTDIR:-${WORKhafs}/intercom/chgres}
-  export DATA=${WORKhafs}/atm_ic
-else
+if [ ${ENSDA} = YES ]; then
   export ENSID=${ENSID:-001}
   export OUTDIR=${OUTDIR:-${WORKhafs}/intercom/chgres_ens/mem${ENSID}}
   export DATA=${WORKhafs}/atm_ic_ens/mem${ENSID}
+elif [ ${FGAT_MODEL} = gdas ]; then
+  export OUTDIR=${OUTDIR:-${WORKhafs}/intercom/chgres_fgat${FGAT_HR}}
+  export DATA=${WORKhafs}/atm_ic_fgat${FGAT_HR}
+else
+  export OUTDIR=${OUTDIR:-${WORKhafs}/intercom/chgres}
+  export DATA=${WORKhafs}/atm_ic
 fi
 
 export SCRUBDATA=${SCRUBDATA:-YES}
@@ -62,8 +67,6 @@ cd $DATA
 
 # Execute ex-script
 ${HOMEhafs}/scripts/exhafs_atm_ic.sh
-export err=$?
-exit $err
 
 export KEEPDATA=${KEEPDATA:-YES}
 if [ "${KEEPDATA^^}" != YES ]; then

diff --git a/jobs/JHAFS_ATM_INIT b/jobs/JHAFS_ATM_INIT
@@ -17,8 +17,14 @@ export machine=${WHERE_AM_I:-wcoss_cray}
 export envir=${envir:-prod} # prod, para, test
 export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev
 if [ "${RUN_ENVIR^^}" != NCO ]; then
-  module use ${HOMEhafs}/sorc/hafs_forecast.fd/modulefiles
-  module load ufs_${machine}
+ #module list
+ #module unload intelpython
+ #module use ${HOMEhafs}/sorc/hafs_forecast.fd/modulefiles
+ #if [ "${machine}" = "wcoss_cray" ] || [ "${machine}" = "wcoss_dell_p3" ]; then 
+ #  module load ufs_${machine}
+ #else
+ #  module load ufs_${machine}.intel
+ #fi
   module list
 fi
 
@@ -34,35 +40,79 @@ export COMIN=${COMIN:?}
 export COMOUT=${COMOUT:?}
 export COMhafs=${COMhafs:-${COMOUT}}
 
+export MPISERIAL=${MPISERIAL:-${EXEChafs}/hafs_mpiserial.x}
+
 export CDATE=${CDATE:-${YMDH}}
 export cyc=${cyc:?}
 export STORM=${STORM:-FAKE}
 export STORMID=${STORMID:-00L}
 export ENSDA=${ENSDA:-NO}
+export FGAT_MODEL=${FGAT_MODEL:-gfs}
+export FGAT_HR=${FGAT_HR:-00}
 
-# Deterministic or ensemble
-if [ "${ENSDA}" != YES ]; then
-  export FIXgrid=${FIXgrid:-${WORKhafs}/intercom/grid}
-  export INPdir=${INPdir:-${WORKhafs}/intercom/chgres}
-  export DATA=${WORKhafs}/atm_init
-  export RESTARTout=${COMhafs}/RESTART_init
-else
+export run_init=yes
+if [ ${FGAT_MODEL} = gdas ]; then
+  export CDATE=$(${NDATE} $(awk "BEGIN {print ${FGAT_HR}-6}") $CDATE)
+# export out_prefix=$(echo "${STORM}${STORMID}.${CDATE}" | tr '[A-Z]' '[a-z]')
+  export out_prefix="${STORMID,,}.${CDATE}"
+fi
+
+#===============================================================================
+# forecast
+
+if [ "${ENSDA}" = YES ]; then
   export ENSID=${ENSID:-001}
-  export FIXgrid=${FIXgrid:-${WORKhafs}/intercom/grid_ens}
-  export INPdir=${INPdir:-${WORKhafs}/intercom/chgres_ens/mem${ENSID}}
-  export DATA=${WORKhafs}/atm_init_ens/mem${ENSID}
-  export RESTARTout=${COMhafs}/RESTART_init_ens/mem${ENSID}
+  export FIXgrid=${WORKhafs}/intercom/grid_ens
+  export INPdir=${WORKhafs}/intercom/chgres_ens/mem${ENSID}
+  export DATA=${WORKhafs}/atm_init_ens/mem${ENSID}/forecast
+  export RESTARTout=${WORKhafs}/intercom/RESTART_init_ens/mem${ENSID}
+elif [ ${FGAT_MODEL} = gdas ]; then
+  export FIXgrid=${WORKhafs}/intercom/grid
+  export INPdir=${WORKhafs}/intercom/chgres_fgat${FGAT_HR}
+  export DATA=${WORKhafs}/atm_init_fgat${FGAT_HR}/forecast
+  export RESTARTout=${WORKhafs}/intercom/RESTART_init_fgat${FGAT_HR}
+else
+  export FIXgrid=${WORKhafs}/intercom/grid
+  export INPdir=${WORKhafs}/intercom/chgres
+  export DATA=${WORKhafs}/atm_init/forecast
+  export RESTARTout=${WORKhafs}/intercom/RESTART_init
 fi
 
 #export NHRS=$(echo "${dt_atmos}/3600" | bc -l)
-export NHRS=$(awk "BEGIN {print ${dt_atmos}/3600*2}")
+#export NHRS=$(awk "BEGIN {print ${dt_atmos}/3600*2}")
+export NHRS=$(awk "BEGIN {print ${dt_atmos}/3600}")
+export NHRS_ENS=$(awk "BEGIN {print ${dt_atmos}/3600}")
+#export NBDYHRS=0
 export restart_interval="$(awk "BEGIN {print ${dt_atmos}/3600}") 6"
 export warm_start_opt=0
-export RUN_GSI=NO
+
+export RUN_ATM_MERGE=NO
+export RUN_ATM_MERGE_FGAT=NO
+export RUN_ATM_MERGE_ENS=NO
+export RUN_ATM_VI=NO
+export RUN_ATM_VI_FGAT=NO
+export RUN_ATM_VI_ENS=NO
 export RUN_GSI_VR=NO
+export RUN_GSI_VR_FGAT=NO
 export RUN_GSI_VR_ENS=NO
+export RUN_GSI=NO
 export RUN_ENKF=NO
+export RUN_ANALYSIS_MERGE=NO
+export RUN_ANALYSIS_MERGE_ENS=NO
 export run_ocean=no
+export run_wave=no
+export ccpp_suite_regional=${ccpp_suite_regional_init:-$ccpp_suite_regional}
+export ccpp_suite_glob=${ccpp_suite_glob_init:-$ccpp_suite_glob}
+export ccpp_suite_nest=${ccpp_suite_nest_init:-$ccpp_suite_nest}
+export nstf_n1=${nstf_n1_init:-$nstf_n1}
+export nstf_n2=${nstf_n2_init:-$nstf_n2}
+export nstf_n3=${nstf_n3_init:-$nstf_n3}
+export nstf_n4=${nstf_n4_init:-$nstf_n4}
+export nstf_n5=${nstf_n5_init:-$nstf_n5}
+export glob_layoutx=${glob_layoutx_init:-$glob_layoutx}
+export glob_layouty=${glob_layouty_init:-$glob_layouty}
+export layoutx=${layoutx_init:-$layoutx}
+export layouty=${layouty_init:-$layouty}
 
 export SCRUBDATA=${SCRUBDATA:-YES}
 if [ "${SCRUBDATA}" = YES ]; then
@@ -74,12 +124,149 @@ cd $DATA
 
 # Execute ex-script
 ${HOMEhafs}/scripts/exhafs_forecast.sh
-export err=$?
-exit $err
 
 export KEEPDATA=${KEEPDATA:-YES}
 if [ "${KEEPDATA^^}" != YES ]; then
   rm -rf $DATA
 fi
 
 date
+
+#===============================================================================
+# post
+
+if [ ${ENSDA} = YES ]; then
+  export ENSID=${ENSID:-001}
+  export INPdir=${WORKhafs}/atm_init_ens/mem${ENSID}/forecast
+  export DATA=${WORKhafs}/atm_init_ens/mem${ENSID}/post
+  export COMOUTpost=${WORKhafs}/intercom/atm_init_ens/mem${ENSID}
+  export intercom=${WORKhafs}/intercom/atm_init_ens/mem${ENSID}/post
+  export NHRS_ENS=0
+elif [ ${FGAT_MODEL} = gdas ]; then
+  export INPdir=${WORKhafs}/atm_init_fgat${FGAT_HR}/forecast
+  export DATA=${WORKhafs}/atm_init_fgat${FGAT_HR}/post
+  export COMOUTpost=${WORKhafs}/intercom/atm_init_fgat${FGAT_HR}
+  export intercom=${WORKhafs}/intercom/atm_init_fgat${FGAT_HR}/post
+  export NHRS=0
+else
+  export INPdir=${WORKhafs}/atm_init/forecast
+  export DATA=${WORKhafs}/atm_init/post
+  export COMOUTpost=${WORKhafs}/intercom/atm_init
+  export intercom=${WORKhafs}/intercom/atm_init/post
+  export NHRS=0
+fi
+
+export SENDCOM=${SENDCOM:-YES}
+export SCRUBDATA=${SCRUBDATA:-YES}
+
+if [ "${SCRUBDATA}" = YES ]; then
+  rm -rf $DATA
+  rm -f $INPdir/postf???
+  rm -rf ${intercom}
+fi
+mkdir -p ${COMOUTpost}
+mkdir -p ${intercom}
+mkdir -p $DATA
+cd $DATA
+
+# Execute ex-script
+${HOMEhafs}/scripts/exhafs_atm_post.sh
+
+export KEEPDATA=${KEEPDATA:-YES}
+if [ "${KEEPDATA^^}" != YES ]; then
+  rm -rf $DATA
+fi
+
+date
+
+#===============================================================================
+# product
+
+export OMP_NUM_THREADS=1
+export APRUNC=${APRUNS}
+
+if [ ${ENSDA} = YES ]; then
+  export ENSID=${ENSID:-001}
+  export INPdir=${WORKhafs}/intercom/atm_init_ens/mem${ENSID}/post
+  export DATA=${WORKhafs}/atm_init_ens/mem${ENSID}/product
+  export COMOUTproduct=${WORKhafs}/intercom/atm_init_ens/mem${ENSID}
+  export NHRS_ENS=0
+elif [ ${FGAT_MODEL} = gdas ]; then
+  export INPdir=${WORKhafs}/intercom/atm_init_fgat${FGAT_HR}/post
+  export DATA=${WORKhafs}/atm_init_fgat${FGAT_HR}/product
+  export COMOUTproduct=${WORKhafs}/intercom/atm_init_fgat${FGAT_HR}
+  export NHRS=0
+else
+  export INPdir=${WORKhafs}/intercom/atm_init/post
+  export DATA=${WORKhafs}/atm_init/product
+  export COMOUTproduct=${WORKhafs}/intercom/atm_init
+  export NHRS=0
+fi
+
+export SENDCOM=${SENDCOM:-YES}
+export SCRUBDATA=${SCRUBDATA:-YES}
+
+if [ "${SCRUBDATA}" = YES ]; then
+  rm -rf $DATA
+fi
+
+mkdir -p ${COMOUTproduct}
+mkdir -p $DATA
+cd $DATA
+
+if [ ${gtype} = nest ]; then
+  ngrids=$((${nest_grids} + 1))
+else
+  ngrids=${nest_grids}
+fi
+
+rm -f cmdfile_product
+# Loop for grids/domains to prepare the cmdfile
+for ng in $(seq 1 ${ngrids});
+do
+
+if [[ $ng -eq 1 ]]; then
+  neststr=""
+  tilestr=".tile1"
+else
+  neststr=".nest$(printf '%02d' ${ng})"
+  tilestr=".tile$(printf '%d' ${ng})"
+fi
+gridstr=".grid$(printf '%02d' ${ng})"
+echo "export neststr=$neststr tilestr=${tilestr} gridstr=${gridstr}; ${HOMEhafs}/scripts/exhafs_product.sh > run_product${gridstr}.log 2>&1" >> cmdfile_product
+
+done
+
+chmod +x cmdfile_product
+if [ ${machine} = "wcoss_cray" ]; then
+  export OMP_NUM_THREADS=1
+  export APRUNF="aprun -b -j1 -n2 -N1 -d1 -cc depth cfp"
+  ${APRUNF} cmdfile_product
+else
+  ${APRUNC} ${MPISERIAL} -m cmdfile_product
+fi
+
+for ng in $(seq 1 ${ngrids});
+do
+
+gridstr=".grid$(printf '%02d' ${ng})"
+if grep "PROGRAM GETTRK   HAS ENDED" ./run_product${gridstr}.log ; then
+  echo "INFO: successfully ran run_product${gridstr}"
+else
+  echo "ERROR: failed running run_product${gridstr}"
+  echo "ERROR: exitting..."
+  exit 1
+fi
+
+done
+
+export KEEPDATA=${KEEPDATA:-YES}
+if [ "${KEEPDATA^^}" != YES ]; then
+  rm -rf $DATA
+fi
+
+date
+
+#===============================================================================
+
+exit
diff --git a/jobs/JHAFS_ATM_LBC b/jobs/JHAFS_ATM_LBC
@@ -65,8 +65,6 @@ cd $DATA
 
 # Execute ex-script
 ${HOMEhafs}/scripts/exhafs_atm_lbc.sh
-export err=$?
-exit $err
 
 export KEEPDATA=${KEEPDATA:-YES}
 if [ "${KEEPDATA^^}" != YES ]; then

diff --git a/jobs/JHAFS_ATM_POST b/jobs/JHAFS_ATM_POST
@@ -59,8 +59,8 @@ export SCRUBDATA=${SCRUBDATA:-YES}
 
 if [ "${SCRUBDATA}" = YES ]; then
   rm -rf $DATA
-  rm -f $INPdir/postf???
-  rm -rf ${intercom}
+# rm -f $INPdir/post*f???
+# rm -rf ${intercom}
 fi
 mkdir -p ${COMOUTpost}
 mkdir -p ${intercom}
@@ -69,8 +69,6 @@ cd $DATA
 
 # Execute ex-script
 ${HOMEhafs}/scripts/exhafs_atm_post.sh
-export err=$?
-exit $err
 
 export KEEPDATA=${KEEPDATA:-YES}
 if [ "${KEEPDATA^^}" != YES ]; then