diff --git a/etc/lmod-setup.csh b/etc/lmod-setup.csh index 69cffe5e12..ee7ed10603 100644 --- a/etc/lmod-setup.csh +++ b/etc/lmod-setup.csh @@ -13,7 +13,9 @@ else set L_MACHINE=$1 endif -source /etc/csh.login +if ( "$L_MACHINE" != wcoss2 ) then + source /etc/csh.login +endif if ( "$L_MACHINE" == macos ) then arch=$(uname -m) @@ -53,6 +55,9 @@ else if ( "$L_MACHINE" == odin ) then module --initial_load --no_redirect restore setenv MODULEPATH "/oldscratch/ywang/external/hpc-stack/modulefiles/mpi/intel/2020/cray-mpich/7.7.16:/oldscratch/ywang/external/hpc-stack/modulefiles/compiler/intel/2020:/oldscratch/ywang/external/hpc-stack/modulefiles/core:/oldscratch/ywang/external/hpc-stack/modulefiles/stack:/opt/cray/pe/perftools/21.02.0/modulefiles:/opt/cray/ari/modulefiles:/opt/cray/pe/craype-targets/default/modulefiles:/opt/cray/pe/modulefiles:/opt/cray/modulefiles:/opt/modulefiles" +else if ( "$L_MACHINE" = wcoss2 ) then + module reset + else module purge endif diff --git a/etc/lmod-setup.sh b/etc/lmod-setup.sh index 50979f7dbd..9c330d910c 100644 --- a/etc/lmod-setup.sh +++ b/etc/lmod-setup.sh @@ -14,13 +14,15 @@ else L_MACHINE=$1 fi -[[ ${SHELLOPTS} =~ nounset ]] && has_mu=true || has_mu=false -[[ ${SHELLOPTS} =~ errexit ]] && has_me=true || has_me=false -$has_mu && set +u -$has_me && set +e -source /etc/profile -$has_mu && set -u -$has_me && set -e +if [ "$L_MACHINE" != wcoss2 ]; then + [[ ${SHELLOPTS} =~ nounset ]] && has_mu=true || has_mu=false + [[ ${SHELLOPTS} =~ errexit ]] && has_me=true || has_me=false + $has_mu && set +u + $has_me && set +e + source /etc/profile + $has_mu && set -u + $has_me && set -e +fi if [ "$L_MACHINE" = macos ]; then arch=$(uname -m) @@ -56,6 +58,9 @@ elif [ "$L_MACHINE" = odin ]; then module --initial_load --no_redirect restore export MODULEPATH="/oldscratch/ywang/external/hpc-stack/modulefiles/mpi/intel/2020/cray-mpich/7.7.16:/oldscratch/ywang/external/hpc-stack/modulefiles/compiler/intel/2020:/oldscratch/ywang/external/hpc-stack/modulefiles/core:/oldscratch/ywang/external/hpc-stack/modulefiles/stack:/opt/cray/pe/perftools/21.02.0/modulefiles:/opt/cray/ari/modulefiles:/opt/cray/pe/craype-targets/default/modulefiles:/opt/cray/pe/modulefiles:/opt/cray/modulefiles:/opt/modulefiles" +elif [ "$L_MACHINE" = wcoss2 ]; then + module reset + else module purge fi diff --git a/jobs/JREGIONAL_PRE_POST_STAT b/jobs/JREGIONAL_PRE_POST_STAT index 7d80d3e5eb..7611c6bd60 100755 --- a/jobs/JREGIONAL_PRE_POST_STAT +++ b/jobs/JREGIONAL_PRE_POST_STAT @@ -83,6 +83,17 @@ job_postamble # #----------------------------------------------------------------------- # +# Remove post_complete flag file. +# +#----------------------------------------------------------------------- +# +post_complete_file="${COMIN}/post_${PDY}${cyc}_task_complete.txt" +if [ -f ${post_complete_file} ] ; then + rm_vrfy -f ${post_complete_file} +fi +# +#----------------------------------------------------------------------- +# # Restore the shell options saved at the beginning of this script/func- # tion. # diff --git a/jobs/JREGIONAL_RUN_POST b/jobs/JREGIONAL_RUN_POST index 96c4c04aae..ef783f34da 100755 --- a/jobs/JREGIONAL_RUN_POST +++ b/jobs/JREGIONAL_RUN_POST @@ -138,16 +138,14 @@ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # #----------------------------------------------------------------------- # -if [ "${FCST_LEN_HRS}" = "-1" ]; then - for i_cdate in "${!ALL_CDATES[@]}"; do - if [ "${ALL_CDATES[$i_cdate]}" = "${PDY}${cyc}" ]; then - FCST_LEN_HRS="${FCST_LEN_CYCL_ALL[$i_cdate]}" - break - fi - done +if [ ${#FCST_LEN_CYCL[@]} -gt 1 ]; then + cyc_mod=$(( ${cyc} - ${DATE_FIRST_CYCL:8:2} )) + CYCLE_IDX=$(( ${cyc_mod} / ${INCR_CYCL_FREQ} )) + FCST_LEN_HRS=${FCST_LEN_CYCL[$CYCLE_IDX]} + fcst_len_hrs=$( printf "%03d" "${FCST_LEN_HRS}" ) if [ "${fhr}" = "${fcst_len_hrs}" ]; then - touch "${COMIN}/${TN_RUN_POST}_${PDY}${cyc}_task_complete.txt" + touch "${COMIN}/post_${PDY}${cyc}_task_complete.txt" fi fi # diff --git a/parm/wflow/aqm_all.yaml b/parm/wflow/aqm_all.yaml index 9e1354dc20..39dd7c9dfe 100644 --- a/parm/wflow/aqm_all.yaml +++ b/parm/wflow/aqm_all.yaml @@ -36,7 +36,7 @@ task_nexus_gfs_sfc: datadep_gfs: attrs: age: 00:00:00:05 - text: '&COMINgfs;/gfs.@Y@m@d/@H/atmos' + text: '&COMINgfs;/gfs.@Y@m@d/@H/atmos' streq: left: retro right: '{% if not workflow.DO_REAL_TIME %}retro{% endif %}' @@ -117,12 +117,12 @@ task_aqm_ics_ext: task_aqm_ics: <<: *default_aqm attrs: - cycledefs: cycled + cycledefs: cycled_from_second maxtries: '2' command: '&LOAD_MODULES_RUN_TASK_FP; "aqm_ics" "&JOBSdir;/JREGIONAL_AQM_ICS"' envars: <<: *default_vars - PREV_CYCLE_DIR: '&COMIN_DIR;' + PREV_CYCLE_DIR: '&COMIN_DIR;' join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' dependency: and: @@ -133,11 +133,11 @@ task_aqm_ics: datadep_date_tag_tracer: attrs: age: 00:00:00:05 - text: '&COMIN_DIR;/RESTART/@Y@m@d.@H@M@S.fv_tracer.res.tile1.nc' + text: '&COMIN_DIR;/RESTART/@Y@m@d.@H@M@S.fv_tracer.res.tile1.nc' datadep_tracer: attrs: age: 00:00:00:05 - text: &COMIN_DIR;/RESTART/fv_tracer.res.tile1.nc + text: &COMIN_DIR;/RESTART/fv_tracer.res.tile1.nc task_aqm_lbcs: <<: *default_aqm @@ -154,22 +154,14 @@ task_pre_post_stat: command: '&LOAD_MODULES_RUN_TASK_FP; "pre_post_stat" "&JOBSdir;/JREGIONAL_PRE_POST_STAT"' join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' dependency: - taskdep: - attrs: - task: run_fcst_mem000 - #or_do_post: - # and_inline_post: # If inline post ran, wait on the forecast task to complete - # not: - # taskvalid: - # attrs: - # task: run_post_mem000_f000 - # and_run_post: # If post was meant to run, wait on the whole post metatask - # taskvalid: - # attrs: - # task: run_post_mem000_f000 - # metataskdep: - # attrs: - # metatask: run_post_mem000 + or: + datadep: + attrs: + age: 00:00:00:05 + text: !cycstr '&COMIN_DIR;/post_@Y@m@d@H_task_complete.txt' + metataskdep: + attrs: + metatask: run_ens_post task_post_stat_o3: <<: *default_aqm @@ -187,6 +179,7 @@ task_post_stat_pm25: join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' memory: 120G dependency: + taskdep: attrs: task: pre_post_stat @@ -196,6 +189,7 @@ task_bias_correction_o3: join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' memory: 120G dependency: + taskdep: attrs: task: pre_post_stat @@ -205,6 +199,7 @@ task_bias_correction_pm25: join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' memory: 120G dependency: + taskdep: attrs: task: pre_post_stat diff --git a/parm/wflow/coldstart.yaml b/parm/wflow/coldstart.yaml index aae31a07a7..2c34810f5c 100644 --- a/parm/wflow/coldstart.yaml +++ b/parm/wflow/coldstart.yaml @@ -41,7 +41,7 @@ task_get_extrn_ics: datadep_gfs: attrs: age: 00:00:00:05 - text: '&COMINgfs;/gfs.@Y@m@d/@H/atmos' + text: '&COMINgfs;/gfs.@Y@m@d/@H/atmos' streq: left: retro right: '{% if not workflow.DO_REAL_TIME %}retro{% endif %}' @@ -68,7 +68,7 @@ task_get_extrn_lbcs: datadep_gfs: attrs: age: 00:00:00:05 - text: '&COMINgfs;/gfs.@Y@m@d/@H/atmos' + text: '&COMINgfs;/gfs.@Y@m@d/@H/atmos' streq: left: retro right: '{% if not workflow.DO_REAL_TIME %}retro{% endif %}' diff --git a/parm/wflow/default_workflow.yaml b/parm/wflow/default_workflow.yaml index d882adbcac..c89b221ba8 100644 --- a/parm/wflow/default_workflow.yaml +++ b/parm/wflow/default_workflow.yaml @@ -5,9 +5,9 @@ rocoto: entities: ACCOUNT: '{{ user.ACCOUNT }}' CCPA_OBS_DIR: '{{ platform.CCPA_OBS_DIR }}' - COMIN_DIR: '{% if user.RUN_ENVIR == "nco" %}{{ nco.COMIN_BASEDIR }}/{{ nco.RUN }}.@Y@m@d/@H {% else %}{{ nco.COMIN_BASEDIR }}/@Y@m@d@H{% endif %}' + COMIN_DIR: '{% if user.RUN_ENVIR == "nco" %}{{"{}/{}.@Y@m@d/@H".format(nco.COMIN_BASEDIR,nco.RUN)}}{% else %}{{"{}/@Y@m@d@H".format(workflow.EXPTDIR)}}{% endif %}' COMINgfs: '{{ platform.get("COMINgfs") }}' - FCST_DIR: '{{ nco.DATAROOT }}/run_fcst.{{ workflow.WORKFLOW_ID }}_@Y@m@d@H' + FCST_DIR: '{% if user.RUN_ENVIR == "nco" %}{{"{}/run_fcst_mem#mem#.{}_@Y@m@d@H".format(nco.DATAROOT,workflow.WORKFLOW_ID)}}{% else %}{{"{}/@Y@m@d@H".format(workflow.EXPTDIR)}}{% endif %}' GLOBAL_VAR_DEFNS_FP: '{{ workflow.GLOBAL_VAR_DEFNS_FP }}' JOBSdir: '{{ user.JOBSdir }}' LOAD_MODULES_RUN_TASK_FP: '{{ workflow.LOAD_MODULES_RUN_TASK_FP }}' @@ -31,7 +31,7 @@ rocoto: WARMSTART_CYCLE_DIR: '{{ workflow.WARMSTART_CYCLE_DIR }}' WORKFLOW_ID: '{{ workflow.WORKFLOW_ID }}' attrs: - cyclethrottle: "20" + cyclethrottle: "200" realtime: "F" scheduler: '{{ platform.SCHED }}' taskthrottle: "1000" @@ -40,6 +40,8 @@ rocoto: - !startstopfreq ['{{workflow.DATE_FIRST_CYCL}}', '{{workflow.DATE_FIRST_CYCL}}', '{{workflow.INCR_CYCL_FREQ}}'] forecast: - !startstopfreq ['{{workflow.DATE_FIRST_CYCL}}', '{{workflow.DATE_LAST_CYCL}}', '{{workflow.INCR_CYCL_FREQ}}'] + cycled_from_second: + - !startstopfreq ['{%- if workflow.DATE_FIRST_CYCL != workflow.DATE_LAST_CYCL %}{{ [workflow.DATE_FIRST_CYCL[0:8], "{:02d}".format(workflow.INCR_CYCL_FREQ)]|join }}{%- else %}{{workflow.DATE_FIRST_CYCL}}{%- endif %}', '{{workflow.DATE_LAST_CYCL}}', '{{workflow.INCR_CYCL_FREQ}}'] log: !cycstr '&LOGDIR;/FV3LAM_wflow.{% if user.RUN_ENVIR == "nco" %}{{ workflow.WORKFLOW_ID + "." }}{% endif %}log' tasks: taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml"]|include }}' diff --git a/scripts/exregional_aqm_lbcs.sh b/scripts/exregional_aqm_lbcs.sh index 99da094000..ccf09f52e2 100755 --- a/scripts/exregional_aqm_lbcs.sh +++ b/scripts/exregional_aqm_lbcs.sh @@ -70,7 +70,6 @@ else print_info_msg "$VERBOSE" " All executables will be submitted with command \'${RUN_CMD_AQMLBC}\'." fi - # #----------------------------------------------------------------------- # @@ -88,12 +87,14 @@ cd_vrfy $DATA # #----------------------------------------------------------------------- # -yyyymmdd="${PDY}" -mm="${PDY:4:2}" - +CDATE_MOD=$( $DATE_UTIL --utc --date "${PDY} ${cyc} UTC - ${EXTRN_MDL_LBCS_OFFSET_HRS} hours" "+%Y%m%d%H" ) +yyyymmdd=${CDATE_MOD:0:8} +mm="${CDATE_MOD:4:2}" +hh="${CDATE_MOD:8:2}" -if [ "${FCST_LEN_HRS}" = "-1" ]; then - CYCLE_IDX=$(( ${cyc} / ${INCR_CYCL_FREQ} )) +if [ ${#FCST_LEN_CYCL[@]} -gt 1 ]; then + cyc_mod=$(( ${cyc} - ${DATE_FIRST_CYCL:8:2} )) + CYCLE_IDX=$(( ${cyc_mod} / ${INCR_CYCL_FREQ} )) FCST_LEN_HRS=${FCST_LEN_CYCL[$CYCLE_IDX]} fi LBC_SPEC_FCST_HRS=() @@ -119,7 +120,7 @@ The chemical LBC files do not exist: for hr in 0 ${LBC_SPEC_FCST_HRS[@]}; do fhr=$( printf "%03d" "${hr}" ) if [ -r ${INPUT_DATA}/${NET}.${cycle}${dot_ensmem}.gfs_bndy.tile7.f${fhr}.nc ]; then - ncks -A ${chem_lbcs_fn} ${INPUT_DATA}/${NET}.${cycle}${dot_ensmem}.gfs_bndy.tile7.f${fhr}.nc + ncks -A ${chem_lbcs_fn} ${INPUT_DATA}/${NET}.${cycle}${dot_ensmem}.gfs_bndy.tile7.f${fhr}.nc fi done @@ -136,23 +137,28 @@ fi #----------------------------------------------------------------------- # if [ ${DO_AQM_GEFS_LBCS} = "TRUE" ]; then - - RUN_CYC="${cyc}" - CDATE_MOD=$( $DATE_UTIL --utc --date "${PDY} ${cyc} UTC - ${EXTRN_MDL_LBCS_OFFSET_HRS} hours" "+%Y%m%d%H" ) - PDY_MOD=${CDATE_MOD:0:8} - AQM_GEFS_FILE_CYC=${AQM_GEFS_FILE_CYC:-"${CDATE_MOD:8:2}"} + + AQM_GEFS_FILE_CYC=${AQM_GEFS_FILE_CYC:-"${hh}"} AQM_GEFS_FILE_CYC=$( printf "%02d" "${AQM_GEFS_FILE_CYC}" ) + GEFS_CYC_DIFF=$(( cyc - AQM_GEFS_FILE_CYC )) + if [ "${GEFS_CYC_DIFF}" -lt "0" ]; then + TSTEPDIFF=$( printf "%02d" $(( 24 + ${GEFS_CYC_DIFF} )) ) + else + TSTEPDIFF=$( printf "%02d" ${GEFS_CYC_DIFF} ) + fi + AQM_MOFILE_FN="${AQM_GEFS_FILE_PREFIX}.t${AQM_GEFS_FILE_CYC}z.atmf" - if [ ${DO_REAL_TIME} = "TRUE" ]; then - AQM_MOFILE_FP="${COMINgefs}/gefs.${PDY_MOD}/${AQM_GEFS_FILE_CYC}/chem/sfcsig/${AQM_MOFILE_FN}" + if [ "${DO_REAL_TIME}" = "TRUE" ]; then + AQM_MOFILE_FP="${COMINgefs}/gefs.${yyyymmdd}/${AQM_GEFS_FILE_CYC}/chem/sfcsig/${AQM_MOFILE_FN}" else - AQM_MOFILE_FP="${AQM_GEFS_DIR}/${PDY}/${AQM_GEFS_FILE_CYC}/${AQM_MOFILE_FN}" + AQM_MOFILE_FP="${AQM_GEFS_DIR}/${yyyymmdd}/${AQM_GEFS_FILE_CYC}/${AQM_MOFILE_FN}" fi # Check if GEFS aerosol files exist for hr in 0 ${LBC_SPEC_FCST_HRS[@]}; do - fhr=$( printf "%03d" "${hr}" ) + hr_mod=$(( hr + EXTRN_MDL_LBCS_OFFSET_HRS )) + fhr=$( printf "%03d" "${hr_mod}" ) AQM_MOFILE_FHR_FP="${AQM_MOFILE_FP}${fhr}.nemsio" if [ ! -e "${AQM_MOFILE_FHR_FP}" ]; then print_err_msg_exit "The GEFS file (AQM_MOFILE_FHR_FP) for LBCs does not exist: @@ -160,12 +166,11 @@ if [ ${DO_AQM_GEFS_LBCS} = "TRUE" ]; then fi done - GEFS_CYC_DIFF=$( printf "%02d" "$(( RUN_CYC - AQM_GEFS_FILE_CYC ))" ) NUMTS="$(( FCST_LEN_HRS / LBC_SPEC_INTVL_HRS + 1 ))" cat > gefs2lbc-nemsio.ini <