From b5115c52391786b276c7ffccea96f8c963103ff1 Mon Sep 17 00:00:00 2001
From: George Gayno <George.Gayno@noaa.gov>
Date: Thu, 10 Dec 2020 21:07:06 +0000
Subject: [PATCH] feature/mod_cleanup

Remove unused module files.

The ush/fv3gfs_chgres.sh script uses them.  But that
script runs the old chgres for initializing global runs.
It has since been replaced by the utility in ./util/gdas_init.

Issue #235.
---
 modulefiles/module-setup.sh.inc       | 105 ------
 modulefiles/module_base.hera          |  12 -
 modulefiles/module_base.jet           |  28 --
 modulefiles/module_base.wcoss_c       |  54 ---
 modulefiles/module_base.wcoss_cray    |  48 ---
 modulefiles/module_base.wcoss_dell_p3 |  33 --
 ush/fv3gfs_chgres.sh                  | 495 --------------------------
 ush/load_fv3gfs_modules.sh            |  39 --
 8 files changed, 814 deletions(-)
 delete mode 100644 modulefiles/module-setup.sh.inc
 delete mode 100644 modulefiles/module_base.hera
 delete mode 100644 modulefiles/module_base.jet
 delete mode 100644 modulefiles/module_base.wcoss_c
 delete mode 100644 modulefiles/module_base.wcoss_cray
 delete mode 100644 modulefiles/module_base.wcoss_dell_p3
 delete mode 100755 ush/fv3gfs_chgres.sh
 delete mode 100755 ush/load_fv3gfs_modules.sh

diff --git a/modulefiles/module-setup.sh.inc b/modulefiles/module-setup.sh.inc
deleted file mode 100644
index 983f92898..000000000
--- a/modulefiles/module-setup.sh.inc
+++ /dev/null
@@ -1,105 +0,0 @@
-# Create a test function for sh vs. bash detection.  The name is
-# randomly generated to reduce the chances of name collision.
-__ms_function_name="setup__test_function__$$"
-eval "$__ms_function_name() { /bin/true ; }"
-
-# Determine which shell we are using
-__ms_ksh_test=$( eval '__text="text" ; if [[ $__text =~ ^(t).* ]] ; then printf "%s" ${.sh.match[1]} ; fi' 2> /dev/null | cat )
-__ms_bash_test=$( eval 'if ( set | grep '$__ms_function_name' | grep -v name > /dev/null 2>&1 ) ; then echo t ; fi ' 2> /dev/null | cat )
-
-if [[ ! -z "$__ms_ksh_test" ]] ; then
-    __ms_shell=ksh
-elif [[ ! -z "$__ms_bash_test" ]] ; then
-    __ms_shell=bash
-else
-    # Not bash or ksh, so assume sh.
-    __ms_shell=sh
-fi
-
-if [[ -d /lfs4 ]] ; then
-    # We are on NOAA Jet
-    if ( ! eval module help > /dev/null 2>&1 ) ; then
-        source /apps/lmod/lmod/init/$__ms_shell
-    fi
-    module purge
-elif [[ -d /scratch1 ]] ; then
-    # We are on NOAA Hera
-    if ( ! eval module help > /dev/null 2>&1 ) ; then
-        source /apps/lmod/lmod/init/$__ms_shell
-    fi
-    module purge
-elif [[ -d /gpfs/hps && -e /etc/SuSE-release ]] ; then
-    # We are on NOAA Luna or Surge
-    if ( ! eval module help > /dev/null 2>&1 ) ; then
-       source /opt/modules/default/init/$__ms_shell
-    fi
-    module purge
-    module purge
-    # Workaround until module issues are fixed:
-    unset _LMFILES_
-    unset LOADEDMODULES
-    module use /opt/modulefiles
-    module use /opt/cray/ari/modulefiles
-    module use /opt/cray/craype/default/alt-modulefiles
-    module use /opt/cray/alt-modulefiles
-    module use /gpfs/hps/nco/ops/nwprod/modulefiles
-    module use /gpfs/hps/nco/ops/nwprod/lib/modulefiles
-    module use /usrx/local/prod/modulefiles
-elif [[ -L /usrx && "$( readlink /usrx 2> /dev/null )" =~ dell ]] ; then
-    # We are on NOAA Mars or Venus
-    if ( ! eval module help > /dev/null 2>&1 ) ; then
-        source /usrx/local/prod/lmod/lmod/init/$__ms_shell
-    fi
-    module purge
-elif [[ -d /glade ]] ; then
-    # We are on NCAR Yellowstone
-    if ( ! eval module help > /dev/null 2>&1 ) ; then
-        . /usr/share/Modules/init/$__ms_shell
-    fi
-    module purge
-elif [[ -d /lustre && -d /ncrc ]] ; then
-    # We are on GAEA.
-    if ( ! eval module help > /dev/null 2>&1 ) ; then
-        # We cannot simply load the module command.  The GAEA
-        # /etc/profile modifies a number of module-related variables
-        # before loading the module command.  Without those variables,
-        # the module command fails.  Hence we actually have to source
-        # /etc/profile here.
-        source /etc/profile
-        __ms_source_etc_profile=yes
-    else
-        __ms_source_etc_profile=no
-    fi
-    module purge > /dev/null 2>&1
-    module purge
-    # clean up after purge
-    unset _LMFILES_
-    unset _LMFILES_000
-    unset _LMFILES_001
-    unset LOADEDMODULES
-    module load modules
-    if [[ -d /opt/cray/ari/modulefiles ]] ; then
-        module use -a /opt/cray/ari/modulefiles
-    fi
-    if [[ -d /opt/cray/pe/ari/modulefiles ]] ; then
-        module use -a /opt/cray/pe/ari/modulefiles
-    fi
-    if [[ -d /opt/cray/pe/craype/default/modulefiles ]] ; then
-        module use -a /opt/cray/pe/craype/default/modulefiles
-    fi
-    if [[ -s /etc/opt/cray/pe/admin-pe/site-config ]] ; then
-        source /etc/opt/cray/pe/admin-pe/site-config
-    fi
-    if [[ "$__ms_source_etc_profile" == yes ]] ; then
-      source /etc/profile
-      unset __ms_source_etc_profile
-    fi
-else
-    echo WARNING: UNKNOWN PLATFORM 1>&2
-fi
-
-unset __ms_shell
-unset __ms_ksh_test
-unset __ms_bash_test
-unset $__ms_function_name
-unset __ms_function_name
diff --git a/modulefiles/module_base.hera b/modulefiles/module_base.hera
deleted file mode 100644
index 25c9f4756..000000000
--- a/modulefiles/module_base.hera
+++ /dev/null
@@ -1,12 +0,0 @@
-#%Module######################################################################
-##
-##      FV3GFS prerequisites
-##
-
-module load intel/18.0.5.274
-module load impi/2018.0.4
-module load hpss/hpss
-module load nco/4.7.0
-module use /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles
-module load netcdf_parallel/4.7.4
-module load hdf5_parallel/1.10.6
diff --git a/modulefiles/module_base.jet b/modulefiles/module_base.jet
deleted file mode 100644
index 1a2df69c2..000000000
--- a/modulefiles/module_base.jet
+++ /dev/null
@@ -1,28 +0,0 @@
-#%Module#############################################################
-##
-##      nems prerequisites
-##
-
-proc ModulesHelp {} {
-     puts stderr "The prerequisites for compiling or running FV3 on Jet. "
-}
-
-module load intel/15.0.3.187 impi/2018.4.274 szip hdf5 netcdf4/4.2.1.1
-
-module use /lfs4/HFIP/hfv3gfs/nwprod/NCEPLIBS/modulefiles
-module load bacio/v2.0.2
-module load sp/v2.0.2
-module load ip/v2.0.0
-module load w3nco/v2.0.6
-module load w3emc/v2.3.0
-module load nemsio/v2.2.3
-
-module use /lfs4/HFIP/hwrf-vd/soft/modulefiles
-module load prod_util
-module load grib_util
-module load hpss
-
-# mpiserial on Jet includes a cfp wrapper
-module load mpiserial
-
-module load gempak/7.4.2
diff --git a/modulefiles/module_base.wcoss_c b/modulefiles/module_base.wcoss_c
deleted file mode 100644
index 8b6991239..000000000
--- a/modulefiles/module_base.wcoss_c
+++ /dev/null
@@ -1,54 +0,0 @@
-#%Module######################################################################
-##
-##      FV3GFS prerequisites
-##      Installed by:  Samuel Trahan (Samuel.Trahan@noaa.gov)
-##	Installed on:  2017-July-31
-
-# From default environment
-
-module load modules
-
-module load xt-lsfhpc
-module load ncep
-module load alps
-module load dvs
-module load xpmem
-module load ugni
-module load craype-network-aries
-module load switch
-#module load eswrap
-module load PrgEnv-intel
-module load craype
-#module switch craype-haswell
-
-# Is this needed?
-module load subversion
-
-## WCOSS Cray execution prereqs:
-module load rca
-module load alps
-module load xpmem
-module load gni-headers
-module load udreg
-module load ugni
-module load hpss
-
-module load prod_util
-module load g2tmpl-intel/1.4.0
-module load crtm-intel/2.2.6
-module load iobuf/2.0.7
-module load gempak/7.3.0
-
-module load nco-gnu-sandybridge/4.4.4
-module load NetCDF-intel-sandybridge/4.2
-module load cfp-intel-sandybridge/1.1.0
-setenv USE_CFP YES
-
-# modules used by vrfy
-module load pm5
-module load prod_envir
-module load util_shared/1.0.7
-
-module load gcc/6.3.0
-module unload grib_util/1.0.3
-module load grib_util/1.1.0
diff --git a/modulefiles/module_base.wcoss_cray b/modulefiles/module_base.wcoss_cray
deleted file mode 100644
index aade403c7..000000000
--- a/modulefiles/module_base.wcoss_cray
+++ /dev/null
@@ -1,48 +0,0 @@
-#%Module######################################################################
-##
-##      FV3GFS prerequisites
-
-# From default environment
-
-module load modules
-
-module load xt-lsfhpc
-module load ncep
-module load alps
-module load dvs
-module load xpmem
-module load ugni
-module load craype-network-aries
-module load switch
-#module load eswrap
-module load PrgEnv-intel
-module load craype
-#module switch craype-haswell
-
-# Is this needed?
-module load subversion
-
-## WCOSS Cray execution prereqs:
-module load rca
-module load alps
-module load xpmem
-module load gni-headers
-module load udreg
-module load ugni
-module load hpss
-
-module load prod_util
-module load g2tmpl-intel/1.4.0
-module load crtm-intel/2.2.6
-module load iobuf/2.0.7
-module load gempak/7.3.0
-
-module load nco-gnu-sandybridge/4.4.4
-module load NetCDF-intel-sandybridge/4.2
-module load cfp-intel-sandybridge/1.1.0
-setenv USE_CFP YES
-
-module load gcc/6.3.0
-module unload grib_util/1.0.3
-module load grib_util/1.1.0
-
diff --git a/modulefiles/module_base.wcoss_dell_p3 b/modulefiles/module_base.wcoss_dell_p3
deleted file mode 100644
index af1761978..000000000
--- a/modulefiles/module_base.wcoss_dell_p3
+++ /dev/null
@@ -1,33 +0,0 @@
-#%Module######################################################################
-##
-##      FV3GFS prerequisites
-
-# From default environment
-
-module load ips/18.0.1.163
-module load impi/18.0.1    
-module load lsf/10.1       
-module load EnvVars/1.0.2
-
-module load HPSS/5.0.2.5
-module load prod_util/1.1.0
-module load prod_envir/1.0.2
-module load g2tmpl/1.5.0       
-module load crtm/2.2.6
-module load ESMF/7_1_0r
-module unload grib_util/1.0.6   
-module load grib_util/1.1.0
-
-module load NCO/4.7.0   
-module load CFP/2.0.1
-setenv USE_CFP YES
-
-module use -a /usrx/local/nceplibs/dev/NCEPLIBS/modulefiles
-module load hdf5_parallel/1.10.6
-module load netcdf_parallel/4.7.4
-
-# modules used by vrfy
-module load pm5
-
-module use -a /gpfs/dell1/nco/ops/nwprod/modulefiles/
-module load gempak/7.3.1
diff --git a/ush/fv3gfs_chgres.sh b/ush/fv3gfs_chgres.sh
deleted file mode 100755
index a798ff59a..000000000
--- a/ush/fv3gfs_chgres.sh
+++ /dev/null
@@ -1,495 +0,0 @@
-#!/bin/sh
-
-#------------------------------------------------------------------------
-#
-# Script name: fv3gfs_chgres.sh
-#
-# Description: Creates initial conditions for coldstarting the fv3
-#              forecast model.
-#
-# How to use this script:
-#
-#  - Uncomment the job cards for your machine.  Ensure the
-#    'machine' variable is uncommented.  Ensure job cards for
-#    the other machines are prefixed by "##".  
-#  - Check the job card account.  On Hera, this is 
-#    "#SBATCH -A $account".  On WCOSS, this is "#BSUB -P $account".
-#    You may not have permissions to run under the default account.
-#  - Make sure you have linked the 'fixed' directories using
-#    sorc/link_fixdirs.sh.
-#  - Set the following shell variables:
-#      PSLOT     - Your experiment name
-#      CASE_HIGH - Deterministic resolution
-#      CASE_ENKF - Enkf resolution
-#      PTMP      - Directory were the initial condition files will be 
-#                  placed.
-#      CDUMP     - Set to 'gdas' for cycling with enkf.  Set to 'gfs'
-#                  for free forecast only.  NOTE: only the 'gfs'
-#                  cycle is supported when using fv3gfs data as
-#                  input.
-#      CDATE     - year/month/day/hour of your experiment.
-#      HOMEgfs   - Path to your checkout repository.
-#  - Start the script as follows:
-#      Hera: "sbatch fv3gfs_chgres.sh"
-#      WCOSS Dell and Cray: "cat fv3gfs_chgres.sh | bsub"
-#
-#------------------------------------------------------------------------
-
-#----WCOSS_DELL JOBCARD
-#BSUB -P GFS-DEV
-#BSUB -o log.chgres.%J
-#BSUB -e log.chgres.%J
-#BSUB -J fv3_chgres
-#BSUB -q dev
-#BSUB -M 6000
-#BSUB -W 10:00
-#BSUB -R span[ptile=14]
-#BSUB -n 14
-export machine=WCOSS_DELL_P3
-
-#----WCOSS_CRAY JOBCARD
-##BSUB -L /bin/sh
-##BSUB -P GFS-DEV
-##BSUB -oo log.chgres.%J
-##BSUB -eo log.chgres.%J
-##BSUB -J fv3_chgres
-##BSUB -q dev
-##BSUB -M 2400
-##BSUB -W 10:00
-##BSUB -extsched 'CRAYLINUX[]'
-#export machine=WCOSS_C
-
-#---- Hera JOBCARD
-#---- Submit as: sbatch $script
-##SBATCH -J fv3_chgres_driver
-##SBATCH -A fv3-cpu
-##SBATCH --open-mode=truncate
-##SBATCH -o log.chgres
-##SBATCH -e log.chgres
-##SBATCH --nodes=1
-##SBATCH -q batch
-##SBATCH -t 03:00:00
-#export machine=HERA
-
-set -x
-
-export PSLOT=fv3test
-export CDUMP=gfs
-export CASE_HIGH=C768
-export CASE_ENKF=C384
-export CDATE=2019061000
-
-if [ $machine = WCOSS_C ]; then
- export HOMEgfs=$LS_SUBCWD/..
- export PTMP=${PTMP:-"/gpfs/hps2/ptmp/$USER"}
- export zero_bias_dir=/gpfs/hps3/emc/global/noscrub/emc.glopara/ICS/bias_zero
-elif [ $machine = WCOSS_DELL_P3 ]; then
- export HOMEgfs=$LS_SUBCWD/..
- export PTMP=${PTMP:-"/gpfs/dell2/ptmp/$USER"}
- export zero_bias_dir=/gpfs/hps3/emc/global/noscrub/emc.glopara/ICS/bias_zero
-elif [ $machine = HERA ]; then
- export HOMEgfs=$SLURM_SUBMIT_DIR/..
- export PTMP=${PTMP:-"/scratch2/NCEPDEV/stmp1/$USER"}
- export zero_bias_dir=/scratch1/NCEPDEV/global/glopara/bias_zero
-else 
- echo "Please define HOMEgfs and PTMP for your machine. exit"
- exit
-fi
-
-#-------------------------------------------------------------------------------------------------
-# Perform simple check to ensure user has HOMEgfs set correctly and has linked the 'fix'
-# directories.
-#-------------------------------------------------------------------------------------------------
-
-if [ ! -d ${HOMEgfs}/fix/fix_fv3/$CASE_HIGH ]; then
-  set +x
-  echo FATAL ERROR: CANT FIND FIX DIRECTORIES. 
-  echo ENSURE HOMEgfs PATH IS CORRECT.
-  echo YOUR HOMEgfs SETTING IS: $HOMEgfs
-  echo ALSO ENSURE sorc/link_fixdirs.sh WAS RUN.
-  exit 1
-fi
-
-export NSTSMTH=NO                                  ##apply 9-point smoothing to nsst tref
-export NST_TF_CHG=$HOMEgfs/exec/nst_tf_chg.x
-export ZERO_BIAS=YES                                ##zeroed out all bias and radsat files 
-
-
-export ymd=`echo $CDATE | cut -c 1-8`
-export cyc=`echo $CDATE | cut -c 9-10`
-export yy=`echo $CDATE | cut -c 1-4`
-export mm=`echo $CDATE | cut -c 5-6`
-export dd=`echo $CDATE | cut -c 7-8`
-
-export ROTDIR=$PTMP/$PSLOT
-export RUNDIR=$ROTDIR/chgres
-
-export NODES=1
-export APRUNC=""
-
-. $HOMEgfs/ush/load_fv3gfs_modules.sh
-status=$?
-[[ $status -ne 0 ]] && exit $status
-
-if [ $machine = WCOSS_C ]; then
- export KMP_AFFINITY=disabled
- export OMP_NUM_THREADS_CH=24
- export APRUNC="aprun -n 1 -N 1 -j 1 -d $OMP_NUM_THREADS_CH -cc depth"
- export APRUNTF='aprun -q -j1 -n1 -N1 -d1 -cc depth'
- export SUB=$LS_SUBCWD/../util/sub_wcoss_c
- export ACCOUNT=GFS-DEV
- export QUEUE=dev
- export QUEUE_TRANS=dev_transfer 
-elif [ $machine = WCOSS_DELL_P3 ]; then
- export OMP_NUM_THREADS_CH=14
- export KMP_AFFINITY=disabled
- export APRUNC="time"
- export APRUNTF="time"
- export SUB=$LS_SUBCWD/../util/sub_wcoss_d
- export ACCOUNT=GFS-DEV
- export QUEUE=dev
- export QUEUE_TRANS=dev_transfer 
-elif [ $machine = HERA ]; then
- export APRUNC=time
- export APRUNTF=time
- export OMP_NUM_THREADS_CH=24
- export SUB=$SLURM_SUBMIT_DIR/../util/sub_slurm
- export ACCOUNT=fv3-cpu
- export QUEUE=batch
- export QUEUE_TRANS=batch
-else
- echo "$machine not supported, exit"
- exit
-fi
-
-#----------------------------
-#----------------------------
-#--for high-res
-#----------------------------
-#----------------------------
-
-export CASE=$CASE_HIGH
-export INIDIR=$RUNDIR/$CDUMP/$CASE
-export COMROT=$ROTDIR/${CDUMP}.$ymd/$cyc
-export OUTDIR=$COMROT/INPUT
-export DATA=$INIDIR/stmp
-rm -rf $INIDIR $OUTDIR $DATA
-mkdir -p $INIDIR $OUTDIR  $DATA
-cd $INIDIR ||exit 8
-
-#--------------------------------------------------------------------------
-# See if requested data is in the ops directory.  If not, pull from hpss.
-#--------------------------------------------------------------------------
-
-if [ -s $COMROOT/gfs/prod/${CDUMP}.${ymd}/$cyc ]; then
-
-   if [ $CDUMP = gdas ]; then
-     set +x
-     echo GDAS CYCLE NOT SUPPORTED.
-     echo USE NEW PARALLEL VERSION OF CHGRES.
-     echo CONTACT REPOSITORY MANAGER FOR DETAILS.
-     exit
-   fi
-   atm=./${CDUMP}.t${cyc}z.atmanl.nemsio
-   sfc=./${CDUMP}.t${cyc}z.sfcanl.nemsio
-   nst=$sfc  # for fv3gfs, nst fields stored in surface file
-   for ff in $atm $sfc ; do
-     cp $COMROOT/gfs/prod/${CDUMP}.${ymd}/$cyc/$ff .
-   done
-
-else   ##get data from HPSS archive
-
-  if [ $CDATE -lt 2014050100 ]; then
-
-    HPSSPATH=/NCEPPROD/hpssprod/runhistory/rh$yy/$yy$mm/$yy$mm$dd      ##use operational nems gfs nems gfs ics
-    nst=" "
-    if [ $CDUMP = gdas ]; then
-      tarball_high=com_gfs_prod_${CDUMP}.${CDATE}.tar
-      atm=./gdas1.t${cyc}z.sanl
-      sfc=./gdas1.t${cyc}z.sfcanl
-      biascr=./gdas1.t${cyc}z.abias
-      biascr_pc=" "
-      radstat=./gdas1.t${cyc}z.radstat
-      aircraft_t_bias=" "
-    else
-      tarball_high=com_gfs_prod_${CDUMP}.${CDATE}.anl.tar
-      atm=./gfs.t${cyc}z.sanl
-      sfc=./gfs.t${cyc}z.sfcanl
-    fi
-
-  elif [ $CDATE -le 2017072012 ]; then
-    if [ $CDATE -ge 2016110100 ]; then
-      oldexp=prnemsrn
-    elif [ $CDATE -ge 2016050100 ]; then
-      oldexp=pr4rn_1605
-    elif [ $CDATE -ge 2015121500 ]; then
-      oldexp=pr4rn_1512
-    elif [ $CDATE -ge 2015050200 ]; then
-      oldexp=pr4rn_1505
-    elif [ $CDATE -ge 2014073000 ]; then
-      oldexp=pr4rn_1408
-    elif [ $CDATE -ge 2014050100 ]; then
-      oldexp=pr4rn_1405
-    else
-      echo "NEMS GSM retro ICs do not exit, exit"
-      exit 1
-    fi
-
-    HPSSPATH=/5year/NCEPDEV/emc-global/emc.glopara/WCOSS_C/$oldexp    ##use q3fy17 nems gfs parallel ics
-    tarball_high=${CDATE}${CDUMP}.tar
-    atm=gfnanl.${CDUMP}.$CDATE
-    sfc=sfnanl.${CDUMP}.$CDATE
-    nst=nsnanl.${CDUMP}.$CDATE
-    biascr=biascr.${CDUMP}.$CDATE
-    biascr_pc=biascr_pc.${CDUMP}.$CDATE
-    aircraft_t_bias=aircraft_t_bias.${CDUMP}.$CDATE
-    radstat=radstat.${CDUMP}.$CDATE
-  elif [[ $CDATE -le 2019061118 ]]; then
-    HPSSPATH=/NCEPPROD/hpssprod/runhistory/rh$yy/$yy$mm/$yy$mm$dd      ##use operational nems gfs nems gfs ics
-    if [ $CDUMP = gfs ]; then
-      tarball_high=gpfs_hps_nco_ops_com_gfs_prod_${CDUMP}.${CDATE}.anl.tar
-    else
-      tarball_high=gpfs_hps_nco_ops_com_gfs_prod_${CDUMP}.${CDATE}.tar
-    fi
-    atm=./${CDUMP}.t${cyc}z.atmanl.nemsio
-    sfc=./${CDUMP}.t${cyc}z.sfcanl.nemsio
-    nst=./${CDUMP}.t${cyc}z.nstanl.nemsio
-    biascr=./${CDUMP}.t${cyc}z.abias 
-    biascr_pc=./${CDUMP}.t${cyc}z.abias_pc
-    aircraft_t_bias=./${CDUMP}.t${cyc}z.abias_air  
-    radstat=./${CDUMP}.t${cyc}z.radstat
-  else  # FV3GFS starting 2019061200
-    HPSSPATH=/NCEPPROD/hpssprod/runhistory/rh$yy/$yy$mm/$yy$mm$dd
-    if [ $CDUMP = gfs ]; then
-      tarball_high=gpfs_dell1_nco_ops_com_gfs_prod_${CDUMP}.${yy}${mm}${dd}_${cyc}.${CDUMP}_nemsioa.tar
-    else
-      set +x
-      echo GDAS CYCLE NOT SUPPORTED.
-      echo USE NEW PARALLEL VERSION OF CHGRES.
-      echo CONTACT REPOSITORY MANAGER FOR DETAILS.
-      exit
-    fi
-    atm=./${CDUMP}.$yy$mm$dd/$cyc/${CDUMP}.t${cyc}z.atmanl.nemsio
-    sfc=./${CDUMP}.$yy$mm$dd/$cyc/${CDUMP}.t${cyc}z.sfcanl.nemsio
-  fi
-
-#--extract ICs from hpss
-cat > read_hpss.sh <<EOF1
-
-   . $HOMEgfs/ush/load_fv3gfs_modules.sh   2>>/dev/null
-
-   cd $INIDIR
-   htar -xvf  $HPSSPATH/$tarball_high $atm $sfc $nst  
-   if [[ $CDATE -ge 2019061200 ]]; then
-     ln -fs $atm .
-     ln -fs $sfc .
-   fi
-   if [ $CDUMP = gdas ]; then
-      cd $COMROT 
-      htar -xvf  $HPSSPATH/$tarball_high $biascr $biascr_pc $aircraft_t_bias $radstat
-
-      [[ '$biascr' = ./gdas1.t${cyc}z.abias ]] && mv $biascr ./gdas.t${cyc}z.abias
-      [[ '$radstat' = ./gdas1.t${cyc}z.radstat ]] && mv $radstat ./gdas.t${cyc}z.radstat
-
-      [[ '$biascr' = biascr.${CDUMP}.$CDATE ]] && mv biascr.${CDUMP}.$CDATE  ${CDUMP}.t${cyc}z.abias 
-      [[ '${biascr_pc}' = biascr_pc.${CDUMP}.$CDATE ]] &&  mv biascr_pc.${CDUMP}.$CDATE ${CDUMP}.t${cyc}z.abias_pc
-      [[ '$aircraft_t_bias' = aircraft_t_bias.${CDUMP}.$CDATE ]] && mv aircraft_t_bias.${CDUMP}.$CDATE ${CDUMP}.t${cyc}z.abias_air
-      [[ '$radstat' = radstat.${CDUMP}.$CDATE ]] &&  mv radstat.${CDUMP}.$CDATE  ${CDUMP}.t${cyc}z.radstat
-
-   fi
-EOF1
-chmod u+x read_hpss.sh
-$SUB -a $ACCOUNT -q $QUEUE_TRANS -p 1/1/S -r 1024/1/1 -t 2:00:00 -j read_hpss -o read_hpss.out $INIDIR/read_hpss.sh
-
-#................................................
-fi
-#................................................
-
-testfile=$INIDIR/$sfc                 
-nsleep=0; tsleep=120;  msleep=50
-while test ! -s $testfile -a $nsleep -lt $msleep;do
-  sleep $tsleep; nsleep=`expr $nsleep + 1`
-done
-sleep 300
-
-if [ ! -s $testfile ]; then 
-  echo "$testfile does not exist, exit !"
-  exit 1
-fi
-
-[[ $atm = ./gdas1.t${cyc}z.sanl ]] && mv $INIDIR/$atm $INIDIR/gdas.t${cyc}z.sanl
-[[ $sfc = ./gdas1.t${cyc}z.sfcanl ]] && mv $INIDIR/$sfc $INIDIR/gdas.t${cyc}z.sfcanl
-
-#------------------------------
-if [ $NSTSMTH = "YES" ]; then
-#------------------------------
-mv $nst fnsti
-rm -f tf_chg_parm.input
-cat >tf_chg_parm.input <<EOF1
-&config
-nsmth=3,istyp=0,
-/
-EOF1
-$APRUNTF $NST_TF_CHG <tf_chg_parm.input 
-mv fnsto $nst
-if [ $? -ne 0 ] ; then
-  echo "NST_TF_CHG for $CDUMP $CASE failed. exit"
-  exit 1
-fi
-#------------------------------
-fi
-#------------------------------
-
-$HOMEgfs/ush/global_chgres_driver.sh
-if [ $? -ne 0 ] ; then
- echo "chgres for $CDUMP $CASE failed. exit"
- exit 1
-fi
-
-if [ $CDUMP = "gdas" -a $ZERO_BIAS = "YES" ]; then
-   cd $COMROT
-   for ff in abias abias_air abias_pc radstat; do
-    mv gdas.t${cyc}z.${ff}  gdas.t${cyc}z.${ff}_save
-    cp -p ${zero_bias_dir}/gdas.t18z.${ff} gdas.t${cyc}z.${ff}
-   done
-fi 
-
-[[ $CDUMP = gfs ]] && exit
-[[ $CDATE -lt 2012052100 ]] && exit  # no enkf data before this date
-
-#----------------------------
-#----------------------------
-#--for ENKF 
-#----------------------------
-#----------------------------
-export CASE=$CASE_ENKF
-export INIDIR=$RUNDIR/$CDUMP/$CASE
-rm -rf $INIDIR; mkdir -p $INIDIR 
-cd $INIDIR ||exit 8
-
-if [ $CDATE -lt 2014050100 ]; then
-  HPSSPATH=/NCEPPROD/hpssprod/runhistory/rh$yy/$yy$mm/$yy$mm$dd      ##use operational nems gfs nems gfs ics
-  tarball_enkf_atm=com_gfs_prod_enkf.${ymd}_${cyc}.anl.tar    
-  testfile=$INIDIR/siganl_${CDATE}_mem080
-elif [ $CDATE -le 2017072012 ]; then
-   if [ $CDATE -ge 2016110100 ]; then
-     oldexp=prnemsrn
-   elif [ $CDATE -ge 2016050100 ]; then
-     oldexp=pr4rn_1605
-   elif [ $CDATE -ge 2015121500 ]; then
-     oldexp=pr4rn_1512
-   elif [ $CDATE -ge 2015050200 ]; then
-     oldexp=pr4rn_1505
-   elif [ $CDATE -ge 2014073000 ]; then
-     oldexp=pr4rn_1408
-   elif [ $CDATE -ge 2014050100 ]; then
-     oldexp=pr4rn_1405
-   else
-     echo "NEMS GSM retro ICs do not exit, exit"
-     exit 1
-   fi
-
- HPSSPATH=/5year/NCEPDEV/emc-global/emc.glopara/WCOSS_C/$oldexp    ##use q3fy17 nems gfs parallel ics
- tarball_enkf_atm=${CDATE}gdas.enkf.anl.tar
- tarball_enkf_sfcnst=${CDATE}gdas.enkf.sfcanl.tar
- testfile=$INIDIR/sfcanl_${CDATE}_mem080
-else
- HPSSPATH=/NCEPPROD/hpssprod/runhistory/rh$yy/$yy$mm/$yy$mm$dd      ##use operational nems gfs nems gfs ics
- tarball_enkf_atm=gpfs_hps_nco_ops_com_gfs_prod_enkf.${ymd}_${cyc}.anl.tar    
- testfile=$INIDIR/gdas.t${cyc}z.sfcanl.mem080.nemsio
-fi
-
-#--extract ICs from hpss
-cat > read_hpss.sh <<EOF
-
-   . $HOMEgfs/ush/load_fv3gfs_modules.sh   2>>/dev/null
-
-   cd $INIDIR
-   htar -xvf  $HPSSPATH/$tarball_enkf_atm 
-   if [ $CDATE -le 2017072000 ] && [ $CDATE -gt 2014050100 ]; then
-      htar -xvf  $HPSSPATH/$tarball_enkf_sfcnst 
-   fi
-EOF
-chmod u+x read_hpss.sh
-$SUB -a $ACCOUNT -q $QUEUE_TRANS -p 1/1/S -r 1024/1/1 -t 2:00:00 -j read_hpss -o read_hpss.out $INIDIR/read_hpss.sh
-
-nsleep=0; tsleep=120;  msleep=50
-while test ! -s $testfile -a $nsleep -lt $msleep;do
-  sleep $tsleep; nsleep=`expr $nsleep + 1`
-done
-
-if [ ! -s $testfile ]; then
- echo " $testfile does not exist, exit"
- exit
-fi
-
-#---------------------------
-n=1
-while [ $n -le 80 ]; do
-#---------------------------
-mem=$(printf %03i $n)
-mchar=mem$(printf %03i $n)
-
-
-export COMROT=$ROTDIR/enkfgdas.$ymd/$cyc/$mchar
-export OUTDIR=$COMROT/INPUT
-export DATA=$INIDIR/$mchar
-rm -rf $OUTDIR $DATA
-mkdir -p $OUTDIR $DATA
-
-if [ $CDATE -lt 2014050100 ]; then
-  atm=gdas.t${cyc}z.sanl
-  sfc=gdas.t${cyc}z.sfcanl
-  rm -fr $atm $sfc 
-else
-  atm=${CDUMP}.t${cyc}z.atmanl.nemsio
-  sfc=${CDUMP}.t${cyc}z.sfcanl.nemsio
-  nst=${CDUMP}.t${cyc}z.nstanl.nemsio
-  rm -rf $atm $sfc $nst 
-fi
-
-if [ $CDATE -lt 2014050100 ]; then
-   ln -fs siganl_${CDATE}_$mchar $atm
-   ln -fs sfcanl_${CDATE}_$mchar $sfc
-elif [ $CDATE -le 2017072000 ]; then
-   ln -fs siganl_${CDATE}_$mchar $atm
-   ln -fs sfcanl_${CDATE}_$mchar $sfc
-   ln -fs nstanl_${CDATE}_$mchar $nst
-   cp     nstanl_${CDATE}_$mchar fnsti 
-else
-   ln -fs gdas.t${cyc}z.ratmanl.${mchar}.nemsio $atm
-   ln -fs gdas.t${cyc}z.sfcanl.${mchar}.nemsio $sfc
-   ln -fs gdas.t${cyc}z.nstanl.${mchar}.nemsio $nst
-   cp     gdas.t${cyc}z.nstanl.${mchar}.nemsio fnsti
-fi
-
-if [ $NSTSMTH = "YES" ]; then
-
-rm -f tf_chg_parm.input
-cat >tf_chg_parm.input <<EOF1
-&config
-nsmth=3,istyp=0,
-/
-EOF1
-$APRUNTF $NST_TF_CHG <tf_chg_parm.input
-rm -f $nst
-mv fnsto $nst
-if [ $? -ne 0 ] ; then
-  echo "NST_TF_CHG for $CDUMP $CASE failed. exit"
-  exit 1
-fi
-
-fi # NSTSMTH IS YES
-
-$HOMEgfs/ush/global_chgres_driver.sh
-if [ $? -ne 0 ] ; then
- echo "chgres for enkf $CASE ${mchar} failed. exit"
- exit
-fi
-
-#---------------------------
-n=$((n+1))
-done
-#---------------------------
-
-exit
diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh
deleted file mode 100755
index a8bb23662..000000000
--- a/ush/load_fv3gfs_modules.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/sh
-#set -x
-
-###############################################################
-# Setup runtime environment by loading modules
-ulimit_s=$( ulimit -S -s )
-ulimit -S -s 10000
-
-# Find module command and purge:
-source "$HOMEgfs/modulefiles/module-setup.sh.inc" 
-
-# Load our modules:
-module use "$HOMEgfs/modulefiles" 
-
-if [[ -d /lfs3 ]] ; then
-    # We are on NOAA Jet
-	module load module_base.jet 
-elif [[ -d /scratch1 ]] ; then
-    # We are on NOAA Hera
-	module load module_base.hera
-elif [[ -d /gpfs/hps && -e /etc/SuSE-release ]] ; then
-    # We are on NOAA Luna or Surge
-	module load module_base.wcoss_c 
-elif [[ -L /usrx && "$( readlink /usrx 2> /dev/null )" =~ dell ]] ; then
-    # We are on NOAA Mars or Venus
-	module load module_base.wcoss_dell_p3 
-elif [[ -d /glade ]] ; then
-    # We are on NCAR Yellowstone
-	module load module_base.cheyenne 
-elif [[ -d /lustre && -d /ncrc ]] ; then
-    # We are on GAEA.
-	module load module_base.gaea 
-else
-    echo WARNING: UNKNOWN PLATFORM 
-fi
-
-# Restore stack soft limit:
-ulimit -S -s "$ulimit_s"
-unset ulimit_s