From c1966704826435641b8995ef749c405bc7c2854c Mon Sep 17 00:00:00 2001 From: "Kate.Zhang" Date: Mon, 13 May 2024 17:28:15 +0000 Subject: [PATCH] Merge with May 10 version commit 6a9c1372ecce9e50e4f6e10e56f6e504cde1afe6 (HEAD -> develop, origin/develop, origin/HEAD) Author: TerrenceMcGuinness-NOAA Date: Fri May 10 14:17:13 2024 -0400 Do not use BUILT_semphore to force rebuilds when re-run (#2593) Remove the placement of the `BUILT_semaphore` file after the build in the Jenkins Pipeline and force it to rebuild any changes after a PR is re-ran. --- scripts/exgfs_atmos_grib2_special_npoess.sh | 3 +- scripts/exglobal_archive.sh | 5 +- scripts/exglobal_prep_emissions.py | 25 ++++ ush/forecast_det.sh | 13 +- ush/forecast_postdet.sh | 142 ++++++++------------ ush/forecast_predet.sh | 9 +- ush/hpssarch_gen.sh | 15 ++- 7 files changed, 109 insertions(+), 103 deletions(-) create mode 100755 scripts/exglobal_prep_emissions.py diff --git a/scripts/exgfs_atmos_grib2_special_npoess.sh b/scripts/exgfs_atmos_grib2_special_npoess.sh index 3877b50b77..8d182469ed 100755 --- a/scripts/exgfs_atmos_grib2_special_npoess.sh +++ b/scripts/exgfs_atmos_grib2_special_npoess.sh @@ -153,7 +153,8 @@ for (( fhr=SHOUR; fhr <= FHOUR; fhr = fhr + FHINC )); do # existence of the restart files ############################### export pgm="postcheck" - grib_file="${COM_ATMOS_MASTER}/${RUN}.t${cyc}z.goesmasterf${fhr3}.grb2" + # grib_file="${COM_ATMOS_MASTER}/${RUN}.t${cyc}z.goesmasterf${fhr3}.grb2" + grib_file="${COM_ATMOS_MASTER}/${RUN}.t${cyc}z.special.grb2f${fhr3}" if ! wait_for_file "${grib_file}" "${SLEEP_INT}" "${SLEEP_LOOP_MAX}"; then echo "FATAL ERROR: GOES master grib file ${grib_file} not available after max sleep time" export err=9 diff --git a/scripts/exglobal_archive.sh b/scripts/exglobal_archive.sh index 5842c76b57..acb926d0e6 100755 --- a/scripts/exglobal_archive.sh +++ b/scripts/exglobal_archive.sh @@ -237,7 +237,10 @@ if [[ ${HPSSARCH} = "YES" || ${LOCALARCH} = "YES" ]]; then #gdasocean if [ "${DO_OCN}" = "YES" ]; then - targrp_list="${targrp_list} gdasocean gdasocean_analysis" + targrp_list="${targrp_list} gdasocean" + if [[ "${DO_JEDIOCNVAR}" == "YES" ]]; then + targrp_list="${targrp_list} gdasocean_analysis" + fi fi #gdasice diff --git a/scripts/exglobal_prep_emissions.py b/scripts/exglobal_prep_emissions.py new file mode 100755 index 0000000000..ef0e709142 --- /dev/null +++ b/scripts/exglobal_prep_emissions.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +# exglobal_prep_emissions.py +# This script creates a emissions object +# which perform the pre-processing for aerosol emissions +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs import AerosolEmissions + + +# Initialize root logger +logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the emissions pre-processing task + emissions = AerosolEmissions(config) + emissions.initialize() + emissions.configure() + emissions.execute(emissions.task_config.DATA, emissions.task_config.APRUN) + emissions.finalize() diff --git a/ush/forecast_det.sh b/ush/forecast_det.sh index de2a47c921..e4b9ded3d3 100755 --- a/ush/forecast_det.sh +++ b/ush/forecast_det.sh @@ -6,15 +6,9 @@ UFS_det(){ echo "SUB ${FUNCNAME[0]}: Run type determination for UFS" # Determine if the current cycle is a warm start (based on the availability of restarts) - if [[ "${DOIAU:-}" == "YES" ]]; then - if [[ -f "${COM_ATMOS_RESTART_PREV}/${current_cycle_begin:0:8}.${current_cycle_begin:8:2}0000.coupler.res" ]]; then - warm_start=".true." - fi - else - if [[ -f "${COM_ATMOS_RESTART_PREV}/${current_cycle:0:8}.${current_cycle:8:2}0000.coupler.res" ]]; then - warm_start=".true." - fi - fi + if [[ -f "${COM_ATMOS_RESTART_PREV}/${model_start_date_current_cycle:0:8}.${model_start_date_current_cycle:8:2}0000.coupler.res" ]]; then + warm_start=".true." + fi # If restarts were not available, this is likely a cold start if [[ "${warm_start}" == ".false." ]]; then @@ -30,6 +24,7 @@ UFS_det(){ # Since warm start is false, we cannot do IAU DOIAU="NO" IAU_OFFSET=0 + model_start_date_current_cycle=${current_cycle} # It is still possible that a restart is available from a previous forecast attempt # So we have to continue checking for restarts diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 5f9402ef08..1a910b8405 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -48,11 +48,7 @@ FV3_postdet() { restart_date="${RERUN_DATE}" restart_dir="${DATArestart}/FV3_RESTART" else # "${RERUN}" == "NO" - if [[ "${DOIAU}" == "YES" ]]; then - restart_date="${current_cycle_begin}" - else - restart_date="${current_cycle}" - fi + restart_date="${model_start_date_current_cycle}" restart_dir="${COM_ATMOS_RESTART_PREV}" fi @@ -92,11 +88,10 @@ FV3_postdet() { # Need a coupler.res that is consistent with the model start time if [[ "${DOIAU}" == "YES" ]]; then local model_start_time="${previous_cycle}" - local model_current_time="${current_cycle_begin}" else local model_start_time="${current_cycle}" - local model_current_time="${current_cycle}" fi + local model_current_time="${model_start_date_current_cycle}" rm -f "${DATA}/INPUT/coupler.res" cat >> "${DATA}/INPUT/coupler.res" << EOF 3 (Calendar: no_calendar=0, thirty_day_months=1, julian=2, gregorian=3, noleap=4) @@ -258,13 +253,15 @@ FV3_out() { # Copy the final restart files at the end of the forecast segment # The final restart written at the end of the forecast does not include the valid date # TODO: verify the above statement since RM found that it did! - echo "Copying FV3 restarts for 'RUN=${RUN}' at the end of the forecast segment: ${forecast_end_cycle}" - for fv3_restart_file in "${fv3_restart_files[@]}"; do - restart_file="${forecast_end_cycle:0:8}.${forecast_end_cycle:8:2}0000.${fv3_restart_file}" - ${NCP} "${DATArestart}/FV3_RESTART/${restart_file}" \ - "${COM_ATMOS_RESTART}/${restart_file}" - done - + # TODO: For other components, this is only for gfs/gefs - check to see if this should also have this + if [[ "${COPY_FINAL_RESTARTS}" == "YES" ]]; then + echo "Copying FV3 restarts for 'RUN=${RUN}' at the end of the forecast segment: ${forecast_end_cycle}" + for fv3_restart_file in "${fv3_restart_files[@]}"; do + restart_file="${forecast_end_cycle:0:8}.${forecast_end_cycle:8:2}0000.${fv3_restart_file}" + ${NCP} "${DATArestart}/FV3_RESTART/${restart_file}" \ + "${COM_ATMOS_RESTART}/${restart_file}" + done + fi echo "SUB ${FUNCNAME[0]}: Output data for FV3 copied" } @@ -281,11 +278,7 @@ WW3_postdet() { restart_date="${RERUN_DATE}" restart_dir="${DATArestart}/WW3_RESTART" else - if [[ "${DOIAU}" == "YES" ]]; then - restart_date="${current_cycle_begin}" - else - restart_date="${current_cycle}" - fi + restart_date="${model_start_date_current_cycle}" restart_dir="${COM_WAVE_RESTART_PREV}" fi echo "Copying WW3 restarts for 'RUN=${RUN}' at '${restart_date}' from '${restart_dir}'" @@ -384,11 +377,7 @@ MOM6_postdet() { restart_date="${RERUN_DATE}" else # "${RERUN}" == "NO" restart_dir="${COM_OCEAN_RESTART_PREV}" - if [[ "${DOIAU}" == "YES" ]]; then - restart_date="${current_cycle_begin}" - else - restart_date="${current_cycle}" - fi + restart_date="${model_start_date_current_cycle}" fi # Copy MOM6 ICs @@ -489,11 +478,11 @@ MOM6_out() { # Coarser than 1/2 degree has a single MOM restart local mom6_restart_files mom6_restart_file restart_file mom6_restart_files=(MOM.res.nc) - # 1/4 degree resolution has 4 additional restarts + # 1/4 degree resolution has 3 additional restarts case "${OCNRES}" in "025") local nn - for (( nn = 1; nn <= 4; nn++ )); do + for (( nn = 1; nn <= 3; nn++ )); do mom6_restart_files+=("MOM.res_${nn}.nc") done ;; @@ -501,24 +490,22 @@ MOM6_out() { esac # Copy MOM6 restarts at the end of the forecast segment to COM for RUN=gfs|gefs - local restart_file - if [[ "${RUN}" == "gfs" || "${RUN}" == "gefs" ]]; then - echo "Copying MOM6 restarts for 'RUN=${RUN}' at ${forecast_end_cycle}" - for mom6_restart_file in "${mom6_restart_files[@]}"; do - restart_file="${forecast_end_cycle:0:8}.${forecast_end_cycle:8:2}0000.${mom6_restart_file}" - ${NCP} "${DATArestart}/MOM6_RESTART/${restart_file}" \ - "${COM_OCEAN_RESTART}/${restart_file}" - done + if [[ "${COPY_FINAL_RESTARTS}" == "YES" ]]; then + local restart_file + if [[ "${RUN}" == "gfs" || "${RUN}" == "gefs" ]]; then + echo "Copying MOM6 restarts for 'RUN=${RUN}' at ${forecast_end_cycle}" + for mom6_restart_file in "${mom6_restart_files[@]}"; do + restart_file="${forecast_end_cycle:0:8}.${forecast_end_cycle:8:2}0000.${mom6_restart_file}" + ${NCP} "${DATArestart}/MOM6_RESTART/${restart_file}" \ + "${COM_OCEAN_RESTART}/${restart_file}" + done + fi fi - # Copy restarts at the beginning/middle of the next assimilation cycle to COM for RUN=gdas|enkfgdas|enkfgfs + # Copy restarts for the next cycle for RUN=gdas|enkfgdas|enkfgfs if [[ "${RUN}" =~ "gdas" || "${RUN}" == "enkfgfs" ]]; then local restart_date - if [[ "${DOIAU}" == "YES" ]]; then # Copy restarts at the beginning of the next cycle from DATA to COM - restart_date="${next_cycle_begin}" - else # Copy restarts at the middle of the next cycle from DATA to COM - restart_date="${next_cycle}" - fi + restart_date="${model_start_date_next_cycle}" echo "Copying MOM6 restarts for 'RUN=${RUN}' at ${restart_date}" for mom6_restart_file in "${mom6_restart_files[@]}"; do restart_file="${restart_date:0:8}.${restart_date:8:2}0000.${mom6_restart_file}" @@ -526,7 +513,6 @@ MOM6_out() { "${COM_OCEAN_RESTART}/${restart_file}" done fi - } CICE_postdet() { @@ -539,11 +525,7 @@ CICE_postdet() { seconds=$(to_seconds "${restart_date:8:2}0000") # convert HHMMSS to seconds cice_restart_file="${DATArestart}/CICE_RESTART/cice_model.res.${restart_date:0:4}-${restart_date:4:2}-${restart_date:6:2}-${seconds}.nc" else # "${RERUN}" == "NO" - if [[ "${DOIAU}" == "YES" ]]; then - restart_date="${current_cycle_begin}" - else - restart_date="${current_cycle}" - fi + restart_date="${model_start_date_current_cycle}" cice_restart_file="${COM_ICE_RESTART_PREV}/${restart_date:0:8}.${restart_date:8:2}0000.cice_model.res.nc" fi @@ -554,8 +536,8 @@ CICE_postdet() { # Link iceh_ic file to COM. This is the initial condition file from CICE (f000) # TODO: Is this file needed in COM? Is this going to be used for generating any products? local vdate seconds vdatestr fhr fhr3 interval last_fhr - seconds=$(to_seconds "${current_cycle:8:2}0000") # convert HHMMSS to seconds - vdatestr="${current_cycle:0:4}-${current_cycle:4:2}-${current_cycle:6:2}-${seconds}" + seconds=$(to_seconds "${model_start_date_current_cycle:8:2}0000") # convert HHMMSS to seconds + vdatestr="${model_start_date_current_cycle:0:4}-${model_start_date_current_cycle:4:2}-${model_start_date_current_cycle:6:2}-${seconds}" ${NLN} "${COM_ICE_HISTORY}/${RUN}.ice.t${cyc}z.ic.nc" "${DATA}/CICE_OUTPUT/iceh_ic.${vdatestr}.nc" # Link CICE forecast output files from DATA/CICE_OUTPUT to COM @@ -601,24 +583,22 @@ CICE_out() { ${NCP} "${DATA}/ice_in" "${COM_CONF}/ufs.ice_in" # Copy CICE restarts at the end of the forecast segment to COM for RUN=gfs|gefs - local seconds source_file target_file - if [[ "${RUN}" == "gfs" || "${RUN}" == "gefs" ]]; then - echo "Copying CICE restarts for 'RUN=${RUN}' at ${forecast_end_cycle}" - seconds=$(to_seconds "${forecast_end_cycle:8:2}0000") # convert HHMMSS to seconds - source_file="cice_model.res.${forecast_end_cycle:0:4}-${forecast_end_cycle:4:2}-${forecast_end_cycle:6:2}-${seconds}.nc" - target_file="${forecast_end_cycle:0:8}.${forecast_end_cycle:8:2}0000.cice_model.res.nc" - ${NCP} "${DATArestart}/CICE_RESTART/${source_file}" \ - "${COM_ICE_RESTART}/${target_file}" + if [[ "${COPY_FINAL_RESTARTS}" == "YES" ]]; then + local seconds source_file target_file + if [[ "${RUN}" == "gfs" || "${RUN}" == "gefs" ]]; then + echo "Copying CICE restarts for 'RUN=${RUN}' at ${forecast_end_cycle}" + seconds=$(to_seconds "${forecast_end_cycle:8:2}0000") # convert HHMMSS to seconds + source_file="cice_model.res.${forecast_end_cycle:0:4}-${forecast_end_cycle:4:2}-${forecast_end_cycle:6:2}-${seconds}.nc" + target_file="${forecast_end_cycle:0:8}.${forecast_end_cycle:8:2}0000.cice_model.res.nc" + ${NCP} "${DATArestart}/CICE_RESTART/${source_file}" \ + "${COM_ICE_RESTART}/${target_file}" + fi fi - # Copy restarts at the beginning/middle of the next assimilation cycle to COM for RUN=gdas|enkfgdas|enkfgfs + # Copy restarts for next cycle for RUN=gdas|enkfgdas|enkfgfs if [[ "${RUN}" =~ "gdas" || "${RUN}" == "enkfgfs" ]]; then local restart_date - if [[ "${DOIAU}" == "YES" ]]; then # Copy restarts at the beginning of the next cycle from DATA to COM - restart_date="${next_cycle_begin}" - else # Copy restarts at the middle of the next cycle from DATA to COM - restart_date="${next_cycle}" - fi + restart_date="${model_start_date_next_cycle}" echo "Copying CICE restarts for 'RUN=${RUN}' at ${restart_date}" seconds=$(to_seconds "${restart_date:8:2}0000") # convert HHMMSS to seconds source_file="cice_model.res.${restart_date:0:4}-${restart_date:4:2}-${restart_date:6:2}-${seconds}.nc" @@ -713,11 +693,7 @@ CMEPS_postdet() { seconds=$(to_seconds "${restart_date:8:2}0000") # convert HHMMSS to seconds cmeps_restart_file="${DATArestart}/CMEPS_RESTART/ufs.cpld.cpl.r.${restart_date:0:4}-${restart_date:4:2}-${restart_date:6:2}-${seconds}.nc" else # "${RERUN}" == "NO" - if [[ "${DOIAU}" == "YES" ]]; then - restart_date="${current_cycle_begin}" - else - restart_date="${current_cycle}" - fi + restart_date="${model_start_date_current_cycle}" cmeps_restart_file="${COM_MED_RESTART_PREV}/${restart_date:0:8}.${restart_date:8:2}0000.ufs.cpld.cpl.r.nc" fi @@ -747,26 +723,24 @@ CMEPS_out() { echo "SUB ${FUNCNAME[0]}: Copying output data for CMEPS mediator" # Copy mediator restarts at the end of the forecast segment to COM for RUN=gfs|gefs - echo "Copying mediator restarts for 'RUN=${RUN}' at ${forecast_end_cycle}" - local seconds source_file target_file - seconds=$(to_seconds "${forecast_end_cycle:8:2}"0000) - source_file="ufs.cpld.cpl.r.${forecast_end_cycle:0:4}-${forecast_end_cycle:4:2}-${forecast_end_cycle:6:2}-${seconds}.nc" - target_file="${forecast_end_cycle:0:8}.${forecast_end_cycle:8:2}0000.ufs.cpld.cpl.r.nc" - if [[ -f "${DATArestart}/CMEPS_RESTART/${source_file}" ]]; then - ${NCP} "${DATArestart}/CMEPS_RESTART/${source_file}" \ - "${COM_MED_RESTART}/${target_file}" - else - echo "Mediator restart '${DATArestart}/CMEPS_RESTART/${source_file}' not found." - fi + if [[ "${COPY_FINAL_RESTARTS}" == "YES" ]]; then + echo "Copying mediator restarts for 'RUN=${RUN}' at ${forecast_end_cycle}" + local seconds source_file target_file + seconds=$(to_seconds "${forecast_end_cycle:8:2}"0000) + source_file="ufs.cpld.cpl.r.${forecast_end_cycle:0:4}-${forecast_end_cycle:4:2}-${forecast_end_cycle:6:2}-${seconds}.nc" + target_file="${forecast_end_cycle:0:8}.${forecast_end_cycle:8:2}0000.ufs.cpld.cpl.r.nc" + if [[ -f "${DATArestart}/CMEPS_RESTART/${source_file}" ]]; then + ${NCP} "${DATArestart}/CMEPS_RESTART/${source_file}" \ + "${COM_MED_RESTART}/${target_file}" + else + echo "Mediator restart '${DATArestart}/CMEPS_RESTART/${source_file}' not found." + fi + fi - # Copy restarts at the beginning/middle of the next assimilation cycle to COM for RUN=gdas|enkfgdas|enkfgfs + # Copy restarts for the next cycle to COM for RUN=gdas|enkfgdas|enkfgfs if [[ "${RUN}" =~ "gdas" || "${RUN}" == "enkfgfs" ]]; then local restart_date - if [[ "${DOIAU}" == "YES" ]]; then # Copy restarts at the beginning of the next cycle from DATA to COM - restart_date="${next_cycle_begin}" - else # Copy restarts at the middle of the next cycle from DATA to COM - restart_date="${next_cycle}" - fi + restart_date="${model_start_date_next_cycle}" echo "Copying mediator restarts for 'RUN=${RUN}' at ${restart_date}" seconds=$(to_seconds "${restart_date:8:2}"0000) source_file="ufs.cpld.cpl.r.${restart_date:0:4}-${restart_date:4:2}-${restart_date:6:2}-${seconds}.nc" diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index 7c9c76bcd7..bb4de881f3 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -54,7 +54,14 @@ common_predet(){ current_cycle_begin=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} - ${half_window} hours" +%Y%m%d%H) current_cycle_end=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${half_window} hours" +%Y%m%d%H) next_cycle_begin=$(date --utc -d "${next_cycle:0:8} ${next_cycle:8:2} - ${half_window} hours" +%Y%m%d%H) - next_cycle_end=$(date --utc -d "${next_cycle:0:8} ${next_cycle:8:2} + ${half_window} hours" +%Y%m%d%H) + #Define model start date for current_cycle and next_cycle as the time the forecast will start + if [[ "${DOIAU:-}" == "YES" ]]; then + model_start_date_current_cycle="${current_cycle_begin}" + model_start_date_next_cycle="${next_cycle_begin}" + else + model_start_date_current_cycle=${current_cycle} + model_start_date_next_cycle=${next_cycle} + fi forecast_end_cycle=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${FHMAX} hours" +%Y%m%d%H) FHMIN=${FHMIN:-0} diff --git a/ush/hpssarch_gen.sh b/ush/hpssarch_gen.sh index 1b4329c58f..101745da8e 100755 --- a/ush/hpssarch_gen.sh +++ b/ush/hpssarch_gen.sh @@ -560,13 +560,14 @@ if [[ ${type} == "gdas" ]]; then echo "${COM_MED_RESTART/${ROTDIR}\//}/*" } >> "${DATA}/gdasocean_restart.txt" - { - echo "${COM_OCEAN_ANALYSIS/${ROTDIR}\//}/${head}*" - echo "${COM_OCEAN_ANALYSIS/${ROTDIR}\//}/gdas.t??z.ocngrid.nc" - echo "${COM_OCEAN_ANALYSIS/${ROTDIR}\//}/diags" - echo "${COM_OCEAN_ANALYSIS/${ROTDIR}\//}/yaml" - } >> "${DATA}/gdasocean_analysis.txt" - + if [[ ${DO_JEDIOCNVAR} = "YES" ]]; then + { + echo "${COM_OCEAN_ANALYSIS/${ROTDIR}\//}/${head}*" + echo "${COM_OCEAN_ANALYSIS/${ROTDIR}\//}/gdas.t??z.ocngrid.nc" + echo "${COM_OCEAN_ANALYSIS/${ROTDIR}\//}/diags" + echo "${COM_OCEAN_ANALYSIS/${ROTDIR}\//}/yaml" + } >> "${DATA}/gdasocean_analysis.txt" + fi fi if [[ ${DO_ICE} = "YES" ]]; then