Add coupled files to archive and streamline

The following tar files for ocean and ice output are added to the archive job to be stored on HPSS: - ocn_2D - ocn_3D - ocn_xsect - ocn_daily - ocn_ice_grib2_0p25 - ocn_ice_grib2_0p5 - wavocn - ice - gfs_flux_1p00 Handling of the archival of gaussian grid NetCDF files is changed. The former, unused, and now misnamed setting of SAVEFCSTNEMSIO has been changed to ARCH_GAUSSIAN and is now also used consistently between arch.sh and hpssarch_gen.sh. gfs_pgrb2b remains controlled by this setting as well, though it is not a gaussian grib. Additionally, there are new settings, ARCH_GAUSSIAN_FHMAX and ARCH_GAUSSIAN_FHINC, to control which atm and sfc files are saved. The default values are 36 and 6, maintaining the previous hardcoded values The archive job is also streamlined by replacing a bunch of repeating code. Instead of creating tar files in small loops, a list of all the tarfiles to archive is created, then a single loop executes the archival and error checking. Only the MOS tarfile remains separate (for now). The forecast input.nml has been added to the gfsa tarball. The WW3 grid definitions (ww3_multi*) have been added to the gfswave tarball. Refs: NOAA-EMC#416
WalterKolczynski-NOAA · Dec 2, 2021 · c7a545c · c7a545c
1 parent b83d89f
commit c7a545c
Show file tree

Hide file tree

Showing 4 changed files with 135 additions and 121 deletions.
diff --git a/jobs/rocoto/arch.sh b/jobs/rocoto/arch.sh
@@ -175,59 +175,35 @@ cd $ROTDIR
 
 if [ $CDUMP = "gfs" ]; then
 
-    #for targrp in gfsa gfsb - NOTE - do not check htar error status
-    for targrp in gfsa gfsb; do
-        htar -P -cvf $ATARDIR/$CDATE/${targrp}.tar `cat $ARCH_LIST/${targrp}.txt`
-    done
+    targrp_list="gfsa gfsb"
 
-    #for targrp in gfs_flux gfs_netcdf/nemsio gfs_pgrb2b; do
-    if [ ${SAVEFCSTNEMSIO:-"YES"} = "YES" ]; then
+    if [ ${ARCH_GAUSSIAN:-"YES"} = "YES" ]; then
+        targrp_list="$targrp_list gfs_flux gfs_${format}b gfs_pgrb2b"
         if [ $MODE = "cycled" ]; then
-          targrp_list=`gfs_flux gfs_${format}a gfs_${format}b gfs_pgrb2b`
-        elif [ $MODE = "free" ]; then
-          targrp_list=`gfs_flux gfs_${format}b gfs_pgrb2b`
+          targrp_list="$targrp_list gfs_${format}a"
         fi
-        #for targrp in gfs_flux gfs_${format}a gfs_${format}b gfs_pgrb2b; do
-        for targrp in $targrp_list; do
-            htar -P -cvf $ATARDIR/$CDATE/${targrp}.tar `cat $ARCH_LIST/${targrp}.txt`
-            status=$?
-            if [ $status -ne 0  -a $CDATE -ge $firstday ]; then
-                echo "HTAR $CDATE ${targrp}.tar failed"
-                exit $status
-            fi
-        done
     fi
 
-    #for targrp in gfswave
     if [ $DO_WAVE = "YES" -a "$WAVE_CDUMP" != "gdas" ]; then
-        for targrp in gfswave; do
-            htar -P -cvf $ATARDIR/$CDATE/${targrp}.tar `cat $ARCH_LIST/${targrp}.txt`
-            status=$?
-            if [ $status -ne 0  -a $CDATE -ge $firstday ]; then
-                echo "HTAR $CDATE ${targrp}.tar failed"
-                exit $status
-            fi
-        done
+        targrp_list="$targrp_list gfswave"
+    fi
+
+    if [ $DO_OCEAN = "YES" ]; then
+        targrp_list="$targrp_list ocn_ice_grib2_0p5 ocn_ice_grib2_0p25 ocn_2D ocn_3D ocn_xsect ocn_daily wavocn gfs_flux_1p00"
+    fi
+
+    if [ $DO_ICE = "YES" ]; then
+        targrp_list="$targrp_list ice"
     fi
 
     #for restarts    
     if [ $SAVEFCSTIC = "YES" ]; then
-        htar -P -cvf $ATARDIR/$CDATE/gfs_restarta.tar `cat $ARCH_LIST/gfs_restarta.txt`
-        status=$?
-        if [ $status -ne 0  -a $CDATE -ge $firstday ]; then
-            echo "HTAR $CDATE gfs_restarta.tar failed"
-            exit $status
-        fi
+        targrp_list="$targrp_list gfs_restarta"
     fi
 
     #for downstream products
     if [ $DO_BUFRSND = "YES" -o $WAFSF = "YES" ]; then
-        htar -P -cvf $ATARDIR/$CDATE/gfs_downstream.tar `cat $ARCH_LIST/gfs_downstream.txt`
-        status=$?
-        if [ $status -ne 0  -a $CDATE -ge $firstday ]; then
-            echo "HTAR $CDATE gfs_downstream.tar failed"
-            exit $status
-        fi
+        targrp_list="$targrp_list gfs_downstream"  
     fi
 
     #--save mdl gfsmos output from all cycles in the 18Z archive directory
@@ -239,57 +215,37 @@ if [ $CDUMP = "gfs" ]; then
             exit $status
         fi
     fi
+elif [ $CDUMP = "gdas" ]; then
 
-fi
-
-
-if [ $CDUMP = "gdas" ]; then
-
-    htar -P -cvf $ATARDIR/$CDATE/gdas.tar `cat $ARCH_LIST/gdas.txt`
-    status=$?
-    if [ $status -ne 0  -a $CDATE -ge $firstday ]; then
-        echo "HTAR $CDATE gdas.tar failed"
-        exit $status
-    fi
+    targrp_list="gdas"
 
     #gdaswave
     if [ $DO_WAVE = "YES" ]; then
-        htar -P -cvf $ATARDIR/$CDATE/gdaswave.tar `cat $ARCH_LIST/gdaswave.txt`
-        status=$?
-        if [ $status -ne 0  -a $CDATE -ge $firstday ]; then
-            echo "HTAR $CDATE gdaswave.tar failed"
-            exit $status
-        fi
+        targrp_list="$targrp_list gdaswave"
     fi
 
     if [ $SAVEWARMICA = "YES" -o $SAVEFCSTIC = "YES" ]; then
-        htar -P -cvf $ATARDIR/$CDATE/gdas_restarta.tar `cat $ARCH_LIST/gdas_restarta.txt`
-        status=$?
-        if [ $status -ne 0  -a $CDATE -ge $firstday ]; then
-            echo "HTAR $CDATE gdas_restarta.tar failed"
-            exit $status
-        fi
+        targrp_list="$targrp_list gdas_restarta"
+
         if [ $DO_WAVE = "YES" ]; then
-            htar -P -cvf $ATARDIR/$CDATE/gdaswave_restart.tar `cat $ARCH_LIST/gdaswave_restart.txt`
-            status=$?
-            if [ $status -ne 0  -a $CDATE -ge $firstday ]; then
-                echo "HTAR $CDATE gdaswave_restart.tar failed"
-                exit $status
-            fi
+            targrp_list="$targrp_list gdaswave_restart"
         fi
     fi
 
     if [ $SAVEWARMICB = "YES" -o $SAVEFCSTIC = "YES" ]; then
-        htar -P -cvf $ATARDIR/$CDATE/gdas_restartb.tar `cat $ARCH_LIST/gdas_restartb.txt`
-        status=$?
-        if [ $status -ne 0  -a $CDATE -ge $firstday ]; then
-            echo "HTAR $CDATE gdas_restartb.tar failed"
-            exit $status
-        fi
+        targrp_list="$targrp_list gdas_restartb"
     fi
-
 fi
 
+for targrp in $targrp_list; do
+    htar -P -cvf $ATARDIR/$CDATE/${targrp}.tar $(cat $ARCH_LIST/${targrp}.txt)
+    status=$?
+    if [ $status -ne 0 -a $CDATE -ge $firstday ]; then
+        echo "HTAR $CDATE ${targrp}.tar failed"
+        exit $status
+    fi
+done
+
 ###############################################################
 fi  ##end of HPSS archive
 ###############################################################

diff --git a/parm/config/config.arch b/parm/config/config.arch
@@ -8,6 +8,10 @@ echo "BEGIN: config.arch"
 # Get task specific resources
 . $EXPDIR/config.resources arch
 
+export ARCH_GAUSSIAN=${ARCH_GAUSSIAN:-"YES"}
+export ARCH_GAUSSIAN_FHMAX=${ARCH_GAUSSIAN_FHMAX:-36}
+export ARCH_GAUSSIAN_FHINC=${ARCH_GAUSSIAN_FHINC:-6}
+
 #--online archive of nemsio files for fit2obs verification
 export FITSARC="YES"
 export FHMAX_FITS=132

diff --git a/parm/config/config.defaults.s2sw b/parm/config/config.defaults.s2sw
@@ -39,3 +39,8 @@ DOBNDPNT_WAVE='NO'
 
 # config.waveprep
 IOSRV=1
+
+# config.arch
+export ARCH_OUTPUT_HISTORY="YES"
+export ARCH_HISTORY_FHMAX=${FHMAX_GFS_00}
+export ARCH_HISTORY_FHINC=${FHOUT_GFS}
diff --git a/ush/hpssarch_gen.sh b/ush/hpssarch_gen.sh
@@ -13,7 +13,9 @@ CDATE=${CDATE:-2018010100}
 PDY=$(echo $CDATE | cut -c 1-8)
 cyc=$(echo $CDATE | cut -c 9-10)
 OUTPUT_FILE=${OUTPUT_FILE:-"netcdf"}
-OUTPUT_HISTORY=${OUTPUT_HISTORY:-".true."}
+ARCH_GAUSSIAN=${ARCH_GAUSSIAN:-"YES"}
+ARCH_GAUSSIAN_FHMAX=${ARCH_GAUSSIAN_FHMAX:-36}
+ARCH_GAUSSIAN_FHINC=${ARCH_GAUSSIAN_FHINC:-6}
 SUFFIX=${SUFFIX:-".nc"}
 if [ $SUFFIX = ".nc" ]; then
   format="netcdf"
@@ -36,26 +38,25 @@ if [ $type = "gfs" ]; then
   FHMAX_HF_GFS=${FHMAX_HF_GFS:-120}
   FHOUT_HF_GFS=${FHOUT_HF_GFS:-1}
 
-
   rm -f gfsa.txt
   rm -f gfsb.txt
-  rm -f gfs_pgrb2b.txt
-  rm -f gfs_flux.txt
   rm -f gfs_restarta.txt
   touch gfsa.txt
   touch gfsb.txt
-  touch gfs_pgrb2b.txt
-  touch gfs_flux.txt
   touch gfs_restarta.txt
 
-  if [ $MODE = "cycled" ]; then
-    rm -f gfs_${format}a.txt
-    touch gfs_${format}a.txt
-  fi
-
-  if [ $OUTPUT_HISTORY = ".true." ]; then
+  if [ $ARCH_GAUSSIAN = "YES" ]; then
+    rm -f gfs_pgrb2b.txt
     rm -f gfs_${format}b.txt
+    rm -f gfs_flux.txt
+    touch gfs_pgrb2b.txt
     touch gfs_${format}b.txt
+    touch gfs_flux.txt
+
+    if [ $MODE = "cycled" ]; then
+      rm -f gfs_${format}a.txt
+      touch gfs_${format}a.txt
+    fi
   fi
 
   if [ $DO_DOWN = "YES" ]; then
@@ -68,13 +69,32 @@ if [ $type = "gfs" ]; then
 
   head="gfs.t${cyc}z."
 
-  #..................
-  echo  "${dirname}${head}pgrb2b.0p25.anl                  " >>gfs_pgrb2b.txt
-  echo  "${dirname}${head}pgrb2b.0p25.anl.idx              " >>gfs_pgrb2b.txt
-  echo  "${dirname}${head}pgrb2b.0p50.anl                  " >>gfs_pgrb2b.txt
-  echo  "${dirname}${head}pgrb2b.0p50.anl.idx              " >>gfs_pgrb2b.txt
+  if [ $ARCH_GAUSSIAN = "YES" ]; then
+    echo  "${dirname}${head}pgrb2b.0p25.anl                  " >>gfs_pgrb2b.txt
+    echo  "${dirname}${head}pgrb2b.0p25.anl.idx              " >>gfs_pgrb2b.txt
+    echo  "${dirname}${head}pgrb2b.0p50.anl                  " >>gfs_pgrb2b.txt
+    echo  "${dirname}${head}pgrb2b.0p50.anl.idx              " >>gfs_pgrb2b.txt
+
+    if [ $MODE = "cycled" ]; then
+      echo  "${dirname}${head}atmanl${SUFFIX}            " >>gfs_${format}a.txt
+      echo  "${dirname}${head}sfcanl${SUFFIX}            " >>gfs_${format}a.txt
+      echo  "${dirname}${head}atmi*.nc                   " >>gfs_${format}a.txt
+      echo  "${dirname}${head}dtfanl.nc                  " >>gfs_${format}a.txt
+      echo  "${dirname}${head}loginc.txt                 " >>gfs_${format}a.txt
+    fi
+
+    fh=0
+    while [ $fh -le $ARCH_GAUSSIAN_FHMAX ]; do
+      fhr=$(printf %03i $fh)
+      echo  "${dirname}${head}atmf${fhr}${SUFFIX}        " >>gfs_${format}b.txt
+      echo  "${dirname}${head}sfcf${fhr}${SUFFIX}        " >>gfs_${format}b.txt
+      fh=$((fh+ARCH_GAUSSIAN_FHINC))
+    done
+  fi
 
+  #..................
   echo  "./logs/${CDATE}/gfs*.log                          " >>gfsa.txt
+  echo  "${dirname}/atmos/input.nml                        " >>gfsa.txt
   if [ $MODE = "cycled" ]; then
     echo  "${dirname}${head}gsistat                          " >>gfsa.txt
     echo  "${dirname}${head}nsstbufr                         " >>gfsa.txt
@@ -120,16 +140,18 @@ if [ $type = "gfs" ]; then
   fh=0
   while [ $fh -le $FHMAX_GFS ]; do
     fhr=$(printf %03i $fh)
-    echo  "${dirname}${head}pgrb2b.0p25.f${fhr}             " >>gfs_pgrb2b.txt
-    echo  "${dirname}${head}pgrb2b.0p25.f${fhr}.idx         " >>gfs_pgrb2b.txt
-    if [ -s $ROTDIR/${dirpath}${head}pgrb2b.0p50.f${fhr} ]; then
-       echo  "${dirname}${head}pgrb2b.0p50.f${fhr}         " >>gfs_pgrb2b.txt
-       echo  "${dirname}${head}pgrb2b.0p50.f${fhr}.idx     " >>gfs_pgrb2b.txt
+    if [ $ARCH_GAUSSIAN = "YES" ]; then
+      echo  "${dirname}${head}sfluxgrbf${fhr}.grib2           " >>gfs_flux.txt
+      echo  "${dirname}${head}sfluxgrbf${fhr}.grib2.idx       " >>gfs_flux.txt
+
+      echo  "${dirname}${head}pgrb2b.0p25.f${fhr}             " >>gfs_pgrb2b.txt
+      echo  "${dirname}${head}pgrb2b.0p25.f${fhr}.idx         " >>gfs_pgrb2b.txt
+      if [ -s $ROTDIR/${dirpath}${head}pgrb2b.0p50.f${fhr} ]; then
+         echo  "${dirname}${head}pgrb2b.0p50.f${fhr}         " >>gfs_pgrb2b.txt
+         echo  "${dirname}${head}pgrb2b.0p50.f${fhr}.idx     " >>gfs_pgrb2b.txt
+      fi
     fi
 
-    echo  "${dirname}${head}sfluxgrbf${fhr}.grib2           " >>gfs_flux.txt
-    echo  "${dirname}${head}sfluxgrbf${fhr}.grib2.idx       " >>gfs_flux.txt
-
     echo  "${dirname}${head}pgrb2.0p25.f${fhr}              " >>gfsa.txt
     echo  "${dirname}${head}pgrb2.0p25.f${fhr}.idx          " >>gfsa.txt
     echo  "${dirname}${head}logf${fhr}.txt                  " >>gfsa.txt
@@ -151,27 +173,6 @@ if [ $type = "gfs" ]; then
     fh=$((fh+inc))
   done
 
-
-  #..................
-  if [ $MODE = "cycled" ]; then
-    echo  "${dirname}${head}atmanl${SUFFIX}            " >>gfs_${format}a.txt
-    echo  "${dirname}${head}sfcanl${SUFFIX}            " >>gfs_${format}a.txt
-    echo  "${dirname}${head}atmi*.nc                   " >>gfs_${format}a.txt
-    echo  "${dirname}${head}dtfanl.nc                  " >>gfs_${format}a.txt
-    echo  "${dirname}${head}loginc.txt                 " >>gfs_${format}a.txt
-  fi
-
-  #..................
-  if [ $OUTPUT_HISTORY = ".true." ]; then
-  fh=0
-  while [ $fh -le 36 ]; do
-    fhr=$(printf %03i $fh)
-    echo  "${dirname}${head}atmf${fhr}${SUFFIX}        " >>gfs_${format}b.txt
-    echo  "${dirname}${head}sfcf${fhr}${SUFFIX}        " >>gfs_${format}b.txt
-    fh=$((fh+6))
-  done
-  fi
-
   #..................
   if [ $MODE = "cycled" ]; then
     echo  "${dirname}RESTART/*0000.sfcanl_data.tile1.nc  " >>gfs_restarta.txt
@@ -208,11 +209,59 @@ if [ $type = "gfs" ]; then
     head="gfswave.t${cyc}z."
 
     #...........................
+    echo  "${dirname}/wave/rundata/ww3_multi*   " >>gfswave.txt
     echo "${dirname}gridded/${head}*      " >>gfswave.txt
     echo "${dirname}station/${head}*      " >>gfswave.txt
 
   fi
 
+  if [ $DO_OCEAN = "YES" ]; then
+    dirpath="gfs.${PDY}/${cyc}/ocn/"
+    dirname="./${dirpath}"
+
+    head="gfs.t${cyc}z."
+
+    rm -f gfs_flux_1p00.txt
+    rm -f ocn_ice_grib2_0p5.txt 
+    rm -f ocn_ice_grib2_0p25.txt
+    rm -f ocn_2D.txt
+    rm -f ocn_3D.txt
+    rm -f ocn_xsect.txt
+    rm -f ocn_daily.txt
+    rm -f wavocn.txt
+    touch gfs_flux_1p00.txt
+    touch ocn_ice_grib2_0p5.txt
+    touch ocn_ice_grib2_0p25.txt
+    touch ocn_2D.txt
+    touch ocn_3D.txt
+    touch ocn_xsect.txt
+    touch ocn_daily.txt
+    touch wavocn.txt
+    echo  "${dirname}MOM_input                  " >>ocn_2D.txt
+    echo  "${dirname}ocn_2D*                    " >>ocn_2D.txt
+    echo  "${dirname}ocn_3D*                    " >>ocn_3D.txt
+    echo  "${dirname}ocn*EQ*                    " >>ocn_xsect.txt
+    echo  "${dirname}ocn_daily*                 " >>ocn_daily.txt
+    echo  "${dirname}wavocn*                    " >>wavocn.txt
+    echo  "${dirname}ocn_ice*0p5x0p5.grb2       " >>ocn_ice_grib2_0p5.txt
+    echo  "${dirname}ocn_ice*0p25x0p25.grb2     " >>ocn_ice_grib2_0p25.txt
+    echo  "${dirname}${head}flux.1p00.f???      " >>gfs_flux_1p00.txt
+    echo  "${dirname}${head}flux.1p00.f???.idx  " >>gfs_flux_1p00.txt
+  fi
+
+  if [ $DO_ICE = "YES" ]; then
+    dirpath="gfs.${PDY}/${cyc}/ice/"
+    dirname="./${dirpath}"
+
+    head="gfs.t${cyc}z."
+
+    rm -f ice.txt
+    touch ice.txt
+    echo  "${dirname}ice_in                     " >>ice.txt
+    echo  "${dirname}ice*nc                     " >>ice.txt
+  fi
+
+
 #-----------------------------------------------------
 fi   ##end of gfs
 #-----------------------------------------------------