From 7e87be18597fdea3447ddcdb305c506dfd9efdb4 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Tue, 7 Sep 2021 21:13:52 +0000 Subject: [PATCH 01/57] Update machine-setup.sh for wcoss2. Baseline initial build module for that machine. Fixes #559. --- modulefiles/build.wcoss2_cray.intel | 24 ++++++++++++++++++++++++ sorc/machine-setup.sh | 3 +++ 2 files changed, 27 insertions(+) create mode 100644 modulefiles/build.wcoss2_cray.intel diff --git a/modulefiles/build.wcoss2_cray.intel b/modulefiles/build.wcoss2_cray.intel new file mode 100644 index 000000000..471ca839e --- /dev/null +++ b/modulefiles/build.wcoss2_cray.intel @@ -0,0 +1,24 @@ +#%Module##################################################### +## Build and run module for WCOSS2-Cray +############################################################# + +module load envvar/1.0 +module load cmake/3.20.2 + +module load PrgEnv-intel/8.1.0 +module load craype/2.7.8 +module load intel/19.1.3.304 +module load cray-mpich/8.1.7 + +module load netcdf/4.7.4 +module load esmf/8_1_0_beta_snapshot_27 + +module load bacio/2.4.1 +module load sfcio/1.4.1 +module load w3nco/2.4.1 +module load nemsio/2.5.2 +module load sigio/2.3.2 +module load sp/2.3.3 +module load ip/3.3.3 +module load g2/3.4.1 +module load wgrib2/2.0.8 diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh index ad82266e3..d8294f2e5 100644 --- a/sorc/machine-setup.sh +++ b/sorc/machine-setup.sh @@ -27,6 +27,9 @@ if [[ -d /lfs3 ]] ; then fi target=jet module purge +elif [[ -d /lfs/h1 ]] ; then + target=wcoss2_cray + module purge elif [[ -d /scratch1 ]] ; then # We are on NOAA Hera if ( ! eval module help > /dev/null 2>&1 ) ; then From 47e3ea862cb0df29e87090d6b70411c160517771 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Wed, 8 Sep 2021 16:00:50 +0000 Subject: [PATCH 02/57] Kyle's updates for finding the wgrib2 library. Fixes #559 --- cmake/Findwgrib2.cmake | 23 +++++++++++++++++++++++ sorc/chgres_cube.fd/CMakeLists.txt | 3 +-- 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 cmake/Findwgrib2.cmake diff --git a/cmake/Findwgrib2.cmake b/cmake/Findwgrib2.cmake new file mode 100644 index 000000000..7ceabba23 --- /dev/null +++ b/cmake/Findwgrib2.cmake @@ -0,0 +1,23 @@ +# This module produces the target wgrib2::wgrib2 + +find_path(WGRIB2_INCLUDES wgrib2api.mod) + +find_library(WGRIB2_LIBRARIES libwgrib2.a) + +find_program(WGRIB2_EXE wgrib2) +execute_process(COMMAND ${WGRIB2_EXE} --version OUTPUT_VARIABLE version_str) +string(SUBSTRING ${version_str} 3 5 version) + +mark_as_advanced(WGRIB2_INCLUDES WGRIB2_LIBRARIES) + +add_library(wgrib2::wgrib2 UNKNOWN IMPORTED) + +set_target_properties(wgrib2::wgrib2 PROPERTIES + IMPORTED_LOCATION "${WGRIB2_LIBRARIES}" + INTERFACE_INCLUDE_DIRECTORIES "${WGRIB2_INCLUDES}" + INTERFACE_LINK_LIBRARIES "${WGRIB2_LIBRARIES}") + +find_package_handle_standard_args(wgrib2 + REQUIRED_VARS WGRIB2_LIBRARIES WGRIB2_INCLUDES WGRIB2_EXE + VERSION_VAR version +) diff --git a/sorc/chgres_cube.fd/CMakeLists.txt b/sorc/chgres_cube.fd/CMakeLists.txt index 90101ab0b..7b6ad9388 100644 --- a/sorc/chgres_cube.fd/CMakeLists.txt +++ b/sorc/chgres_cube.fd/CMakeLists.txt @@ -49,8 +49,7 @@ target_link_libraries( sp::sp_d w3nco::w3nco_d esmf - wgrib2::wgrib2_lib - wgrib2::wgrib2_api + wgrib2::wgrib2 MPI::MPI_Fortran NetCDF::NetCDF_Fortran) From 51e9fbd7215bac0c2839b6974cdccc53b6d41dd0 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Wed, 8 Sep 2021 19:54:18 +0000 Subject: [PATCH 03/57] Update link_fixdirs.sh for wcoss2. For now, Kate will host the fixed files in her own directory. Fixes #559. --- fix/link_fixdirs.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/fix/link_fixdirs.sh b/fix/link_fixdirs.sh index bc5ea6f41..d848dc6d0 100755 --- a/fix/link_fixdirs.sh +++ b/fix/link_fixdirs.sh @@ -9,17 +9,17 @@ machine=${2} if [ $# -lt 2 ]; then set +x echo '***ERROR*** must specify two arguements: (1) RUN_ENVIR, (2) machine' - echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet | orion | s4 )' + echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 | cray | dell | hera | jet | orion | s4 )' exit 1 fi if [ $RUN_ENVIR != emc -a $RUN_ENVIR != nco ]; then set +x echo '***ERROR*** unsupported run environment' - echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet | orion | s4 )' + echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 | cray | dell | hera | jet | orion | s4 )' exit 1 fi -if [ $machine != cray -a $machine != hera -a $machine != dell -a $machine != jet -a $machine != orion -a $machine != s4 ]; then +if [ $machine != wcoss2 -a $machine != cray -a $machine != hera -a $machine != dell -a $machine != jet -a $machine != orion -a $machine != s4 ]; then set +x echo '***ERROR*** unsupported machine' echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet | orion | s4 )' @@ -45,6 +45,8 @@ elif [ $machine = "jet" ]; then FIX_DIR="/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix" elif [ $machine = "orion" ]; then FIX_DIR="/work/noaa/global/glopara/fix" +elif [ $machine = "wcoss2" ]; then + FIX_DIR="/lfs/h2/emc/eib/noscrub/Kate.Friedman/glopara/fix_nco_gfsv16" elif [ $machine = "s4" ]; then FIX_DIR="/data/prod/glopara/fix" fi From f3588cfc29f2f4355d0681980d2872e250bf2df1 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Wed, 8 Sep 2021 20:22:11 +0000 Subject: [PATCH 04/57] Load additional libraries from the stack instead of letting cmake find another version. Fixes #559 --- modulefiles/build.wcoss2_cray.intel | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/modulefiles/build.wcoss2_cray.intel b/modulefiles/build.wcoss2_cray.intel index 471ca839e..c9f1ba6ea 100644 --- a/modulefiles/build.wcoss2_cray.intel +++ b/modulefiles/build.wcoss2_cray.intel @@ -10,6 +10,12 @@ module load craype/2.7.8 module load intel/19.1.3.304 module load cray-mpich/8.1.7 +module load libjpeg/9c +module load jasper/2.0.25 +module load zlib/1.2.11 +module load libpng/1.6.37 + +module load hdf5/1.10.6 module load netcdf/4.7.4 module load esmf/8_1_0_beta_snapshot_27 @@ -22,3 +28,6 @@ module load sp/2.3.3 module load ip/3.3.3 module load g2/3.4.1 module load wgrib2/2.0.8 + +# for mpiexec command?? +##module load cray-pals/1.0.12 From 7d946c29899d845be483e452ec5c5b8f95f528bd Mon Sep 17 00:00:00 2001 From: George Gayno Date: Wed, 8 Sep 2021 20:57:54 +0000 Subject: [PATCH 05/57] Add grid generation driver script for WCOSS2. Fixes #559. --- driver_scripts/driver_grid.wcoss2.sh | 158 +++++++++++++++++++++++++++ modulefiles/build.wcoss2_cray.intel | 4 +- 2 files changed, 160 insertions(+), 2 deletions(-) create mode 100755 driver_scripts/driver_grid.wcoss2.sh diff --git a/driver_scripts/driver_grid.wcoss2.sh b/driver_scripts/driver_grid.wcoss2.sh new file mode 100755 index 000000000..f0d853893 --- /dev/null +++ b/driver_scripts/driver_grid.wcoss2.sh @@ -0,0 +1,158 @@ +#!/bin/bash + +#PBS -o log +#PBS -e log +#PBS -q debug +#PBS -A GFS-DEV +#PBS -l walltime=00:05:00 +#PBS -N make_grid +#PBS -l select=1:ncpus=48 + +#----------------------------------------------------------------------- +# Driver script to create a cubic-sphere based model grid on WCOSS2. +# +# Produces the following files (netcdf, each tile in separate file): +# 1) 'mosaic' and 'grid' files containing lat/lon and other +# records that describe the model grid. +# 2) 'oro' files containing land mask, terrain and gravity +# wave drag fields. +# 3) Surface climo fields, such as soil type, vegetation +# greenness and albedo. +# +# Note: The sfc_climo_gen program only runs with an +# mpi task count that is a multiple of six. This is +# an ESMF library requirement. Large grids may require +# tasks spread across multiple nodes. The orography code +# benefits from threads. +# +# To run, do the following: +# +# 1) Set "C" resolution, "res" - Example: res=96. +# 2) Set grid type ("gtype"). Valid choices are +# "uniform" - global uniform grid +# "stretch" - global stretched grid +# "nest" - global stretched grid with nest +# "regional_gfdl" - stand-alone gfdl regional grid +# "regional_esg" - stand-alone extended Schmidt gnomonic +# (esg) regional grid +# 3) For "uniform" and "regional_gfdl" grids - to include lake +# fraction and depth, set "add_lake" to true, and the +# "lake_cutoff" value. +# 4) For "stretch" and "nest" grids, set the stretching factor - +# "stretch_fac", and center lat/lon of highest resolution +# tile - "target_lat" and "target_lon". +# 5) For "nest" grids, set the refinement ratio - "refine_ratio", +# the starting/ending i/j index location within the parent +# tile - "istart_nest", "jstart_nest", "iend_nest", "jend_nest" +# 6) For "regional_gfdl" grids, set the "halo". Default is three +# rows/columns. +# 7) For "regional_esg" grids, set center lat/lon of grid, +# - "target_lat/lon" - the i/j dimensions - "i/jdim", the +# x/y grid spacing - "delx/y", and halo. +# 8) Set working directory - TEMP_DIR - and path to the repository +# clone - home_dir. +# 9) Check settings for 'make_gsl_orog' and 'veg_type_src' +# below. +# 10) Submit script: "cat $script | bsub". +# 11) All files will be placed in "out_dir". +# +#----------------------------------------------------------------------- + +cd $PBS_O_WORKDIR + +source ../sorc/machine-setup.sh > /dev/null 2>&1 +module use ../modulefiles +module load build.$target.intel +module list + +#----------------------------------------------------------------------- +# Set grid specs here. +#----------------------------------------------------------------------- + +export gtype=uniform # 'uniform', 'stretch', 'nest', + # 'regional_gfdl', 'regional_esg' +export make_gsl_orog=false # 'true' if user needs 'oro' files for GSL + # orographic drag suite +export veg_type_src="modis.igbp.0.05" # veg type data. + # For viirs-based vegetation type data, set to: + # 1) "viirs.igbp.0.05" for global 5km data + # 2) "viirs.igbp.0.1" for global 10km data + # 3) "viirs.igbp.0.03" for global 3km data + # 4) "viirs.igbp.conus.0.01" for regional 1km data + # For the modis-based data, set to: + # 1) "modis.igbp.0.05" for global 5km data + # 2) "modis.igbp.0.03" for global 3km data + # 3) "modis.igbp.conus.0.01" for regional 1km data + +if [ $gtype = uniform ]; then + export res=96 + export add_lake=false # Add lake frac and depth to orography data. + export lake_cutoff=0.20 # lake frac < lake_cutoff ignored when add_lake=T +elif [ $gtype = stretch ]; then + export res=96 + export stretch_fac=1.5 # Stretching factor for the grid + export target_lon=-97.5 # Center longitude of the highest resolution tile + export target_lat=35.5 # Center latitude of the highest resolution tile +elif [ $gtype = nest ] || [ $gtype = regional_gfdl ]; then + export add_lake=false # Add lake frac and depth to orography data. + export lake_cutoff=0.20 # lake frac < lake_cutoff ignored when add_lake=T + export res=768 + export stretch_fac=1.5 # Stretching factor for the grid + export target_lon=-97.5 # Center longitude of the highest resolution tile + export target_lat=38.5 # Center latitude of the highest resolution tile + export refine_ratio=3 # The refinement ratio + export istart_nest=123 # Starting i-direction index of nest grid in parent tile supergrid + export jstart_nest=331 # Starting j-direction index of nest grid in parent tile supergrid + export iend_nest=1402 # Ending i-direction index of nest grid in parent tile supergrid + export jend_nest=1194 # Ending j-direction index of nest grid in parent tile supergrid + export halo=3 # Lateral boundary halo +elif [ $gtype = regional_esg ] ; then + export res=-999 # equivalent resolution is computed + export target_lon=-97.5 # Center longitude of grid + export target_lat=35.5 # Center latitude of grid + export idim=301 # Dimension of grid in 'i' direction + export jdim=200 # Dimension of grid in 'j' direction + export delx=0.0585 # Grid spacing (in degrees) in the 'i' direction + # on the SUPERGRID (which has twice the resolution of + # the model grid). The physical grid spacing in the 'i' + # direction is related to delx as follows: + # distance = 2*delx*(circumf_Earth/360 deg) + export dely=0.0585 # Grid spacing (in degrees) in the 'j' direction. + export halo=3 # number of row/cols for halo +fi + +#----------------------------------------------------------------------- +# Check paths. +# home_dir - location of repository. +# TEMP_DIR - working directory. +# out_dir - where files will be placed upon completion. +#----------------------------------------------------------------------- + +export home_dir=$PBS_O_WORKDIR/.. +export TEMP_DIR=/lfs/h2/emc/stmp/$LOGNAME/fv3_grid.$gtype +export out_dir=/lfs/h2/emc/stmp/$LOGNAME/my_grids + +#----------------------------------------------------------------------- +# Should not need to change anything below here unless you want to +# to change the job card for the number of tasks to use. Then, +# you will need to check APRUN_SFC and OMP_NUM_THREADS. +#----------------------------------------------------------------------- + +set -x + +export APRUN=time +export APRUN_SFC="mpiexec -n 24 -ppn 24" +export OMP_NUM_THREADS=24 # orog code worked best with 24 threads. +export OMP_STACKSIZE=2048m +export machine=WCOSS2_CRAY + +ulimit -a +ulimit -s unlimited + +#----------------------------------------------------------------------- +# Start script. +#----------------------------------------------------------------------- + +$home_dir/ush/fv3gfs_driver_grid.sh + +exit diff --git a/modulefiles/build.wcoss2_cray.intel b/modulefiles/build.wcoss2_cray.intel index c9f1ba6ea..a40f1ef1b 100644 --- a/modulefiles/build.wcoss2_cray.intel +++ b/modulefiles/build.wcoss2_cray.intel @@ -29,5 +29,5 @@ module load ip/3.3.3 module load g2/3.4.1 module load wgrib2/2.0.8 -# for mpiexec command?? -##module load cray-pals/1.0.12 +# for mpiexec command +module load cray-pals/1.0.12 From a61adfb81b9e2b62ec2b54282ba7fb59e2941a76 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Thu, 9 Sep 2021 12:56:14 +0000 Subject: [PATCH 06/57] Update vcoord_gen utility for wcoss2. Fixes #559 --- util/vcoord_gen/run.sh | 2 +- util/vcoord_gen/run.wcoss2.sh | 56 +++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100755 util/vcoord_gen/run.wcoss2.sh diff --git a/util/vcoord_gen/run.sh b/util/vcoord_gen/run.sh index b0d5c3aff..bf8ed0852 100755 --- a/util/vcoord_gen/run.sh +++ b/util/vcoord_gen/run.sh @@ -3,7 +3,7 @@ #------------------------------------------------------------------------------- # # Generate a hybrid coordinate interface profile. On WCOSS-Cray, use -# 'run.cray.sh'. +# 'run.cray.sh'. On WCOSS2, use 'run.wcoss2.sh'. # # Build the repository using the ./build_all.sh script before running. # diff --git a/util/vcoord_gen/run.wcoss2.sh b/util/vcoord_gen/run.wcoss2.sh new file mode 100755 index 000000000..7f2e096fc --- /dev/null +++ b/util/vcoord_gen/run.wcoss2.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +#PBS -l walltime=00:02:00 +#PBS -o log +#PBS -e log +#PBS -N vcoord +#PBS -q debug +#PBS -A GFS-DEV +#PBS -l select=1:ncpus=2:mem=1MB + +#------------------------------------------------------------------------------- +# +# Generate a hybrid coordinate interface profile on WCOSS2. +# +# Build the repository using the ./build_all.sh script before running. +# +# Output 'ak' and 'bk' values are placed in $outfile. +# +# To run this script, do: 'qsub $script' +# +#------------------------------------------------------------------------------- + +cd $PBS_O_WORKDIR + +set -x + +source ../../sorc/machine-setup.sh > /dev/null 2>&1 +module use ../../modulefiles +module load build.$target.intel +module list + +outfile="./global_hyblev.txt" + +levs=128 # integer number of levels +lupp=88 # integer number of levels below pupp +pbot=100000.0 # real nominal surface pressure (Pa) +psig=99500.0 # real nominal pressure where coordinate changes + # from pure sigma (Pa) +ppre=7000.0 # real nominal pressure where coordinate changes + # to pure pressure (Pa) +pupp=7000.0 # real nominal pressure where coordinate changes + # to upper atmospheric profile (Pa) +ptop=0.0 # real pressure at top (Pa) +dpbot=240.0 # real coordinate thickness at bottom (Pa) +dpsig=1200.0 # real thickness of zone within which coordinate changes + # to pure sigma (Pa) +dppre=18000.0 # real thickness of zone within which coordinate changes + # to pure pressure (Pa) +dpupp=550.0 # real coordinate thickness at pupp (Pa) +dptop=1.0 # real coordinate thickness at top (Pa) + +rm -f $outfile + +echo $levs $lupp $pbot $psig $ppre $pupp $ptop $dpbot $dpsig $dppre $dpupp $dptop | ../../exec/vcoord_gen > $outfile + +exit From da9db591f63c14f793c8a0aacc83dbaa8d8e85ae Mon Sep 17 00:00:00 2001 From: George Gayno Date: Thu, 9 Sep 2021 13:09:23 +0000 Subject: [PATCH 07/57] Create ice_blend regression test driver script for wcoss2. Fixes #559. --- reg_tests/ice_blend/driver.wcoss2.sh | 53 ++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100755 reg_tests/ice_blend/driver.wcoss2.sh diff --git a/reg_tests/ice_blend/driver.wcoss2.sh b/reg_tests/ice_blend/driver.wcoss2.sh new file mode 100755 index 000000000..af60196a8 --- /dev/null +++ b/reg_tests/ice_blend/driver.wcoss2.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# +# Run ice_blend consistency test on WCOSS2. +# +# Set $DATA to your working directory. Set the project code (PBS -A) +# and queue (PBS -q) as appropriate. +# +# Invoke the script as follows: qsub $script +# +# Log output is placed in consistency.log. A summary is +# placed in summary.log +# +# The test fails when its output does not match the baseline file +# as determined by the 'cmp' command. The baseline file is +# stored in HOMEreg. +# +#----------------------------------------------------------------------------- + +#PBS -l walltime=00:02:00 +#PBS -o consistency.log +#PBS -e consistency.log +#PBS -N iceb_regt +#PBS -q debug +#PBS -A GFS-DEV +#PBS -l select=1:ncpus=2:mem=2500MB + +cd $PBS_O_WORKDIR + +source ../../sorc/machine-setup.sh > /dev/null 2>&1 +module use ../../modulefiles +module load build.$target.intel +module load grib_util/1.2.2 +module list + +set -x + +export DATA="${WORK_DIR:-/lfs/h2/emc/stmp/$LOGNAME}" +export DATA="${DATA}/reg-tests/ice-blend" + +#----------------------------------------------------------------------------- +# Should not have to change anything below. +#----------------------------------------------------------------------------- + +export HOMEreg=/lfs/h2/emc/eib/noscrub/George.Gayno/ufs_utils.git/reg_tests/ice_blend +export HOMEgfs=$PBS_O_WORKDIR/../.. + +rm -fr $DATA + +./ice_blend.sh + +exit 0 From 301f5d1f7d2bfaa04ac00ef198d35ae9310e52d4 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Thu, 9 Sep 2021 13:30:32 +0000 Subject: [PATCH 08/57] Create sfc_climo_gen utility script for wcoss2. Fixes #559. --- util/sfc_climo_gen/run.wcoss2.sh | 84 ++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100755 util/sfc_climo_gen/run.wcoss2.sh diff --git a/util/sfc_climo_gen/run.wcoss2.sh b/util/sfc_climo_gen/run.wcoss2.sh new file mode 100755 index 000000000..39232e52c --- /dev/null +++ b/util/sfc_climo_gen/run.wcoss2.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +#------------------------------------------------------------ +# Run the sfc_climo_gen program stand-alone on WCOSS2 using +# pre-exiting 'grid' and 'orography files. +# +# To run, type: 'qsub $script' +#------------------------------------------------------------ + +#PBS -o log +#PBS -e log +#PBS -q debug +#PBS -A GFS-DEV +#PBS -N grid_fv3 +#PBS -l walltime=00:10:00 +#PBS -l select=1:ncpus=48 + +set -x + +export BASE_DIR=$PBS_O_WORKDIR/../.. + +source ${BASE_DIR}/sorc/machine-setup.sh > /dev/null 2>&1 +module use ${BASE_DIR}/modulefiles +module load build.$target.intel +module list + +#------------------------------------- +# Set model resolution. +#------------------------------------- + +export res=384 + +#------------------------------------- +# Where the model "grid", "mosaic" and "oro" files reside. +#------------------------------------- + +export FIX_FV3=${BASE_DIR}/fix/fix_fv3_gmted2010/C${res} + +#------------------------------------- +# Uncomment for regional grids. +#------------------------------------- + +##HALO=3 +##export GRIDTYPE=regional + +#------------------------------------- +# Choose which virrs data to use. +#------------------------------------- + +export veg_type_src="viirs.igbp.0.05" # Use global 0.05-degree viirs data +#export veg_type_src="viirs.igbp.0.1" # Use global 0.1-degree viirs data +#export veg_type_src="viirs.igbp.0.03" # Use global 0.03-degree viirs data +#export veg_type_src="viirs.igbp.conus.0.01" # Use CONUS 0.01-degree virrs data. Do not + # use for global grids. + +#------------------------------------- +# Set working directory and directory where output files will be saved. +#------------------------------------- + +export WORK_DIR=/lfs/h2/emc/stmp/$LOGNAME/work.sfc +export SAVE_DIR=/lfs/h2/emc/stmp/$LOGNAME/sfc.C${res} + +#------------------------------------- +# Should not have to touch anything below here. +#------------------------------------- + +if [[ $GRIDTYPE = "regional" ]]; then + HALO=$(( $HALO + 1 )) + export HALO + ln -fs $FIX_FV3/C${res}_grid.tile7.halo${HALO}.nc $FIX_FV3/C${res}_grid.tile7.nc + ln -fs $FIX_FV3/C${res}_oro_data.tile7.halo${HALO}.nc $FIX_FV3/C${res}_oro_data.tile7.nc +fi + +export input_sfc_climo_dir=${BASE_DIR}/fix/fix_sfc_climo +export APRUN_SFC="mpiexec -n 24 -ppn 24" + +ulimit -a +ulimit -s unlimited + +rm -fr $WORK_DIR $SAVE_DIR + +${BASE_DIR}/ush/sfc_climo_gen.sh + +exit From 8a25192de8d1e40165e7f4d84be48491c5d281eb Mon Sep 17 00:00:00 2001 From: George Gayno Date: Thu, 9 Sep 2021 13:58:42 +0000 Subject: [PATCH 09/57] Add snow2mdl regression test script for wcoss2. Fixes #559. --- reg_tests/snow2mdl/driver.wcoss2.sh | 53 +++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100755 reg_tests/snow2mdl/driver.wcoss2.sh diff --git a/reg_tests/snow2mdl/driver.wcoss2.sh b/reg_tests/snow2mdl/driver.wcoss2.sh new file mode 100755 index 000000000..6d43d558f --- /dev/null +++ b/reg_tests/snow2mdl/driver.wcoss2.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# +# Run snow2mdl consistency test on WCOSS2. +# +# Set $DATA to your working directory. Set the project code (PBS -A) +# and queue (PBS -q) as appropriate. +# +# Invoke the script as follows: qsub $script +# +# Log output is placed in consistency.log. A summary is +# placed in summary.log +# +# The test fails when its output does not match the baseline file +# as determined by the 'cmp' command. The baseline file is +# stored in HOMEreg. +# +#----------------------------------------------------------------------------- + +#PBS -l walltime=00:05:00 +#PBS -o consistency.log +#PBS -e consistency.log +#PBS -N s2m_regt +#PBS -q debug +#PBS -A GFS-DEV +#PBS -l select=1:ncpus=2:mem=2500MB + +cd $PBS_O_WORKDIR + +source ../../sorc/machine-setup.sh > /dev/null 2>&1 +module use ../../modulefiles +module load build.$target.intel +module load grib_util/1.2.2 +module list + +set -x + +export DATA="${WORK_DIR:-/lfs/h2/emc/stmp/$LOGNAME}" +export DATA="${DATA}/reg-tests/snow2mdl" + +#----------------------------------------------------------------------------- +# Should not have to change anything below. +#----------------------------------------------------------------------------- + +export HOMEreg=/lfs/h2/emc/eib/noscrub/George.Gayno/ufs_utils.git/reg_tests/snow2mdl +export HOMEgfs=$PWD/../.. + +rm -fr $DATA + +./snow2mdl.sh + +exit 0 From 1812a0f5486733cded2ba4024d0b9693853b8d5d Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Tue, 14 Sep 2021 15:09:43 +0000 Subject: [PATCH 10/57] Add global_cycle regression test script for wcoss2. Fixes #559. --- reg_tests/global_cycle/driver.wcoss2.sh | 76 +++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100755 reg_tests/global_cycle/driver.wcoss2.sh diff --git a/reg_tests/global_cycle/driver.wcoss2.sh b/reg_tests/global_cycle/driver.wcoss2.sh new file mode 100755 index 000000000..bed6a21c2 --- /dev/null +++ b/reg_tests/global_cycle/driver.wcoss2.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# +# Run global_cycle consistency test on WCOSS2. +# +# Set $WORK_DIR to your working directory. Set the project code +# and queue as appropriate. +# +# Invoke the script from the command line as follows: ./$script +# +# Log output is placed in consistency.log??. A summary is +# placed in summary.log +# +# A test fails when its output does not match the baseline files +# as determined by the 'nccmp' utility. This baseline files are +# stored in HOMEreg. +# +#----------------------------------------------------------------------------- + +set -x + +compiler=${compiler:-"intel"} + +source ../../sorc/machine-setup.sh > /dev/null 2>&1 +module use ../../modulefiles +module load build.$target.$compiler +module list + +WORK_DIR="${WORK_DIR:-/lfs/h2/emc/stmp/$LOGNAME}" + +PROJECT_CODE="${PROJECT_CODE:-GFS-DEV}" +QUEUE="${QUEUE:-dev}" + +#----------------------------------------------------------------------------- +# Should not have to change anything below. +#----------------------------------------------------------------------------- + +DATA_DIR="${WORK_DIR}/reg-tests/global-cycle" + +export HOMEreg=/lfs/h2/emc/eib/noscrub/George.Gayno/ufs_utils.git/reg_tests/global_cycle + +export OMP_NUM_THREADS_CY=2 + +export APRUNCY="mpiexec -n 6 -ppn 6 --cpu-bind core --depth 2" + +export NWPROD=$PWD/../.. + +reg_dir=$PWD + +LOG_FILE=consistency.log +rm -f ${LOG_FILE}* + +export DATA="${DATA_DIR}/test1" +export COMOUT=$DATA +TEST1=$(qsub -V -o ${LOG_FILE}01 -e ${LOG_FILE}01 -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ + -N c768.fv3gfs -l select=1:ncpus=12:mem=8GB $PWD/C768.fv3gfs.sh) + +export DATA="${DATA_DIR}/test2" +export COMOUT=$DATA +TEST2=$(qsub -V -o ${LOG_FILE}02 -e ${LOG_FILE}02 -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ + -N c768.lndincsoil -l select=1:ncpus=12:mem=1GB $PWD/C768.lndincsoil.sh) + +export DATA="${DATA_DIR}/test3" +export COMOUT=$DATA +TEST3=$(qsub -V -o ${LOG_FILE}03 -e ${LOG_FILE}03 -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ + -N c768.lndincsnow -l select=1:ncpus=12:mem=1GB $PWD/C768.lndincsnow.sh) + +qsub -V -o ${LOG_FILE} -e ${LOG_FILE} -q $QUEUE -A $PROJECT_CODE -l walltime=00:01:00 \ + -N cycle_summary -l select=1:ncpus=2:mem=100MB -W depend=afterok:$TEST1:$TEST2:$TEST3 << EOF +#!/bin/bash +cd $reg_dir +grep -a '<<<' ${LOG_FILE}?? > summary.log +EOF + +exit From a4c80f30349d4d895845b2221de4a8e9036771d1 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Wed, 15 Sep 2021 13:56:52 +0000 Subject: [PATCH 11/57] Add grid_gen regression test driver script for wcoss2. Fixes #559. --- reg_tests/grid_gen/driver.wcoss2.sh | 103 ++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100755 reg_tests/grid_gen/driver.wcoss2.sh diff --git a/reg_tests/grid_gen/driver.wcoss2.sh b/reg_tests/grid_gen/driver.wcoss2.sh new file mode 100755 index 000000000..67b3d6a3d --- /dev/null +++ b/reg_tests/grid_gen/driver.wcoss2.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# +# Run grid generation consistency tests on WCOSS2. +# +# Set WORK_DIR to your working directory. Set the PROJECT_CODE and QUEUE +# as appropriate. +# +# Invoke the script with no arguments. A series of daily- +# chained jobs will be submitted. To check the queue, type: +# "qstat -u USERNAME". +# +# Log output from the suite will be in LOG_FILE. Once the suite +# has completed, a summary is placed in SUM_FILE. +# +# A test fails when its output does not match the baseline files as +# determined by the "nccmp" utility. The baseline files are stored in +# $HOMEreg. +# +#----------------------------------------------------------------------------- + +compiler=${compiler:-"intel"} + +source ../../sorc/machine-setup.sh > /dev/null 2>&1 +module use ../../modulefiles +module load build.$target.$compiler +module list + +set -x + +export WORK_DIR="${WORK_DIR:-/lfs/h2/emc/stmp/$LOGNAME}" +export WORK_DIR="${WORK_DIR}/reg-tests/grid-gen" +QUEUE="${QUEUE:-dev}" +PROJECT_CODE="${PROJECT_CODE:-GFS-DEV}" + +#----------------------------------------------------------------------------- +# Should not have to change anything below here. +#----------------------------------------------------------------------------- + +LOG_FILE=consistency.log +rm -f ${LOG_FILE} +SUM_FILE=summary.log +export home_dir=$PWD/../.. +export APRUN=time +export APRUN_SFC="mpiexec -n 30 -ppn 30" +export OMP_STACKSIZE=2048m +export machine=WCOSS2 +export HOMEreg=/lfs/h2/emc/eib/noscrub/George.Gayno/ufs_utils.git/reg_tests/grid_gen/baseline_data +this_dir=$PWD + +ulimit -a + +rm -fr $WORK_DIR + +export OMP_NUM_THREADS=30 +export OMP_PLACES=cores + +#----------------------------------------------------------------------------- +# C96 uniform grid +#----------------------------------------------------------------------------- + +TEST1=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:08:00 \ + -N c96.uniform -l select=1:ncpus=60:mem=40GB $PWD/c96.uniform.sh) + +#----------------------------------------------------------------------------- +# C96 uniform grid using viirs vegetation data. +#----------------------------------------------------------------------------- + +TEST2=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:08:00 \ + -N c96.viirs.vegt -l select=1:ncpus=60:mem=40GB -W depend=afterok:$TEST1 $PWD/c96.viirs.vegt.sh) + +#----------------------------------------------------------------------------- +# gfdl regional grid +#----------------------------------------------------------------------------- + +TEST3=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:08:00 \ + -N gfdl.regional -l select=1:ncpus=60:mem=40GB -W depend=afterok:$TEST2 $PWD/gfdl.regional.sh) + +#----------------------------------------------------------------------------- +# esg regional grid +#----------------------------------------------------------------------------- + +TEST4=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:08:00 \ + -N esg.regional -l select=1:ncpus=60:mem=40GB -W depend=afterok:$TEST3 $PWD/esg.regional.sh) + +#----------------------------------------------------------------------------- +# Regional GSL gravity wave drag test. +#----------------------------------------------------------------------------- + +TEST5=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:08:00 \ + -N rsg.gsl.gwd -l select=1:ncpus=60:mem=40GB -W depend=afterok:$TEST4 $PWD/regional.gsl.gwd.sh) + +#----------------------------------------------------------------------------- +# Create summary log. +#----------------------------------------------------------------------------- + +qsub -V -o ${LOG_FILE} -e ${LOG_FILE} -q $QUEUE -A $PROJECT_CODE -l walltime=00:01:00 \ + -N grid_summary -l select=1:ncpus=2:mem=100MB -W depend=afterok:$TEST5 << EOF +#!/bin/bash +cd ${this_dir} +grep -a '<<<' $LOG_FILE > $SUM_FILE +EOF From ee51e7d14db71ac68768cd9b6ae82adcbf0dd5dc Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 16 Sep 2021 15:00:21 +0000 Subject: [PATCH 12/57] Minor updates to global_cycle reg test script. Fixes #559 --- reg_tests/global_cycle/driver.wcoss2.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/reg_tests/global_cycle/driver.wcoss2.sh b/reg_tests/global_cycle/driver.wcoss2.sh index bed6a21c2..445ab3cc4 100755 --- a/reg_tests/global_cycle/driver.wcoss2.sh +++ b/reg_tests/global_cycle/driver.wcoss2.sh @@ -41,6 +41,7 @@ DATA_DIR="${WORK_DIR}/reg-tests/global-cycle" export HOMEreg=/lfs/h2/emc/eib/noscrub/George.Gayno/ufs_utils.git/reg_tests/global_cycle export OMP_NUM_THREADS_CY=2 +export OMP_PLACES=cores export APRUNCY="mpiexec -n 6 -ppn 6 --cpu-bind core --depth 2" @@ -54,17 +55,17 @@ rm -f ${LOG_FILE}* export DATA="${DATA_DIR}/test1" export COMOUT=$DATA TEST1=$(qsub -V -o ${LOG_FILE}01 -e ${LOG_FILE}01 -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N c768.fv3gfs -l select=1:ncpus=12:mem=8GB $PWD/C768.fv3gfs.sh) + -N c768.fv3gfs -l select=1:ncpus=24:mem=8GB $PWD/C768.fv3gfs.sh) export DATA="${DATA_DIR}/test2" export COMOUT=$DATA TEST2=$(qsub -V -o ${LOG_FILE}02 -e ${LOG_FILE}02 -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N c768.lndincsoil -l select=1:ncpus=12:mem=1GB $PWD/C768.lndincsoil.sh) + -N c768.lndincsoil -l select=1:ncpus=24:mem=1GB $PWD/C768.lndincsoil.sh) export DATA="${DATA_DIR}/test3" export COMOUT=$DATA TEST3=$(qsub -V -o ${LOG_FILE}03 -e ${LOG_FILE}03 -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N c768.lndincsnow -l select=1:ncpus=12:mem=1GB $PWD/C768.lndincsnow.sh) + -N c768.lndincsnow -l select=1:ncpus=24:mem=1GB $PWD/C768.lndincsnow.sh) qsub -V -o ${LOG_FILE} -e ${LOG_FILE} -q $QUEUE -A $PROJECT_CODE -l walltime=00:01:00 \ -N cycle_summary -l select=1:ncpus=2:mem=100MB -W depend=afterok:$TEST1:$TEST2:$TEST3 << EOF From 773ae5d99a909b5f84cf7ea72291cc61140428a6 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 16 Sep 2021 15:50:40 +0000 Subject: [PATCH 13/57] Add chgres_cube regression test driver script for wcoss2. Fixes #559. --- reg_tests/chgres_cube/README | 20 +-- reg_tests/chgres_cube/driver.wcoss2.sh | 219 +++++++++++++++++++++++++ 2 files changed, 228 insertions(+), 11 deletions(-) create mode 100755 reg_tests/chgres_cube/driver.wcoss2.sh diff --git a/reg_tests/chgres_cube/README b/reg_tests/chgres_cube/README index 424accd4b..23d03f292 100644 --- a/reg_tests/chgres_cube/README +++ b/reg_tests/chgres_cube/README @@ -7,18 +7,16 @@ To run the consistency tests: 1) Build chgres_cube program. Go to ./sorc and invoke 'build_all.sh' with no arguments. -2) Invoke driver script for your machine. See script - prolog for details. Supported machines are: - - Hera (driver.hera.sh) - - WCOSS-Cray (driver.cray.sh) - - WCOSS-Dell (driver.dell.sh) - - Jet (driver.jet.sh) - - Orion (driver.orion.sh) +2) Set the fixed directories. Go to ./fix and + invoke the script. +3) Invoke driver script for your machine. See script + prolog for details. -A series of daisy-chained tests will run. A test fails -if the output files differ from the baseline set of files -as determined by the 'nccmp' utility. +A series of tests will run. A test fails if the output +files differ from the baseline set of files as determined +by the 'nccmp' utility. -Log output from the tests will be in "regression.log". +Log output from each test will be placed in its own +logfile - "consistency.log??". A summary of results will be in "summary.log". diff --git a/reg_tests/chgres_cube/driver.wcoss2.sh b/reg_tests/chgres_cube/driver.wcoss2.sh new file mode 100755 index 000000000..e09955094 --- /dev/null +++ b/reg_tests/chgres_cube/driver.wcoss2.sh @@ -0,0 +1,219 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# +# Run the chgres_cube consistency tests on WCOSS2. +# +# Set WORK_DIR to a general working location outside the UFS_UTILS directory. +# The exact working directory (OUTDIR) will be WORK_DIR/reg_tests/chgres-cube. +# +# Set the PROJECT_CODE and QUEUE as appropriate. +# +# Invoke the script with no arguments. To check the queue, type: +# "qstat -u USERNAME". +# +# The run output will be stored in OUTDIR. Log output will be placed +# in LOG_FILE??. Once the suite has completed, a summary is placed +# in SUM_FILE. +# +# A test fails when its output does not match the baseline files as +# determined by the "nccmp" utility. The baseline files are stored in +# HOMEreg. +# +#----------------------------------------------------------------------------- + +set -x + +compiler=${compiler:-"intel"} + +source ../../sorc/machine-setup.sh > /dev/null 2>&1 +module use ../../modulefiles +module load build.$target.$compiler +module list + +export OUTDIR="${WORK_DIR:-/lfs/h2/emc/stmp/$LOGNAME}" +export OUTDIR="${OUTDIR}/reg-tests/chgres-cube" + +PROJECT_CODE="${PROJECT_CODE:-GFS-DEV}" +QUEUE="${QUEUE:-dev}" + +#----------------------------------------------------------------------------- +# Should not have to change anything below here. HOMEufs is the root +# directory of your UFS_UTILS clone. HOMEreg contains the input data +# and baseline data for each test. +#----------------------------------------------------------------------------- + +export HOMEufs=$PWD/../.. + +export HOMEreg=/lfs/h2/emc/eib/noscrub/George.Gayno/ufs_utils.git/reg_tests/chgres_cube + +LOG_FILE=consistency.log +SUM_FILE=summary.log +rm -f $LOG_FILE* $SUM_FILE + +export OMP_STACKSIZE=1024M + +export NCCMP=${NCCMP:-nccmp} +rm -fr $OUTDIR + +this_dir=$PWD + +#----------------------------------------------------------------------------- +# Initialize C96 using FV3 warm restart files. +#----------------------------------------------------------------------------- + +LOG_FILE=consistency.log01 +export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" +TEST1=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ + -N c96.fv3.restart -l select=1:ncpus=12:ompthreads=1:mem=10GB $PWD/c96.fv3.restart.sh) + +#----------------------------------------------------------------------------- +# Initialize C192 using FV3 tiled history files. +#----------------------------------------------------------------------------- + +LOG_FILE=consistency.log02 +export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" +TEST2=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ + -N c192.fv3.history -l select=1:ncpus=12:ompthreads=1:mem=10GB $PWD/c192.fv3.history.sh) + +#----------------------------------------------------------------------------- +# Initialize C96 using FV3 gaussian nemsio files. +#----------------------------------------------------------------------------- + +LOG_FILE=consistency.log03 +export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" +TEST3=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ + -N c96.fv3.nemsio -l select=1:ncpus=12:ompthreads=1:mem=45GB $PWD/c96.fv3.nemsio.sh) + +#----------------------------------------------------------------------------- +# Initialize C96 using spectral GFS sigio/sfcio files. +#----------------------------------------------------------------------------- + +LOG_FILE=consistency.log04 +export OMP_PLACES=cores +export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core --depth 4" +TEST4=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:10:00 \ + -N c96.gfs.sigio -l select=1:ncpus=48:ompthreads=4:mem=45GB $PWD/c96.gfs.sigio.sh) + +#----------------------------------------------------------------------------- +# Initialize C96 using spectral GFS gaussian nemsio files. +#----------------------------------------------------------------------------- + +LOG_FILE=consistency.log05 +export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" +TEST5=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ + -N c96.gfs.nemsio -l select=1:ncpus=12:ompthreads=1:mem=35GB $PWD/c96.gfs.nemsio.sh) + +#----------------------------------------------------------------------------- +# Initialize regional C96 using FV3 gaussian nemsio files. +#----------------------------------------------------------------------------- + +LOG_FILE=consistency.log06 +export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" +TEST6=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ + -N c96.regional -l select=1:ncpus=12:ompthreads=1:mem=35GB $PWD/c96.regional.sh) + +#----------------------------------------------------------------------------- +# Initialize C96 using FV3 gaussian netcdf files. +#----------------------------------------------------------------------------- + +LOG_FILE=consistency.log07 +export APRUN="mpiexec -n 12 -ppn 12 --cpu-bind core" +TEST7=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ + -N c96.fv3.netcdf -l select=1:ncpus=24:ompthreads=1:mem=80GB $PWD/c96.fv3.netcdf.sh) + +#----------------------------------------------------------------------------- +# Initialize global C192 using GFS GRIB2 files. +#----------------------------------------------------------------------------- + +LOG_FILE=consistency.log08 +export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" +TEST8=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ + -N c192.gfs.grib2 -l select=1:ncpus=12:ompthreads=1:mem=15GB $PWD/c192.gfs.grib2.sh) + +#----------------------------------------------------------------------------- +# Initialize CONUS 25-KM USING GFS GRIB2 files. +#----------------------------------------------------------------------------- + +LOG_FILE=consistency.log09 +export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" +TEST9=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ + -N 25km.conus.gfs.grib2.conus -l select=1:ncpus=12:ompthreads=1:mem=15GB $PWD/25km.conus.gfs.grib2.sh) + +#----------------------------------------------------------------------------- +# Initialize CONUS 3-KM USING HRRR GRIB2 file WITH GFS PHYSICS. +#----------------------------------------------------------------------------- + +LOG_FILE=consistency.log10 +export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" +TEST10=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ + -N 3km.conus.hrrr.gfssdf.grib2.conus -l select=1:ncpus=12:ompthreads=1:mem=40GB $PWD/3km.conus.hrrr.gfssdf.grib2.sh) + +#----------------------------------------------------------------------------- +# Initialize CONUS 3-KM USING HRRR GRIB2 file WITH GSD PHYSICS AND SFC VARS FROM FILE. +#----------------------------------------------------------------------------- + +LOG_FILE=consistency.log11 +export APRUN="mpiexec -n 12 -ppn 12 --cpu-bind core" +TEST11=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ + -N 3km.conus.hrrr.newsfc.grib2.conus -l select=1:ncpus=24:ompthreads=1:mem=50GB $PWD/3km.conus.hrrr.newsfc.grib2.sh) + +#----------------------------------------------------------------------------- +# Initialize CONUS 13-KM USING NAM GRIB2 file WITH GFS PHYSICS . +#----------------------------------------------------------------------------- + +LOG_FILE=consistency.log12 +export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" +TEST12=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ + -N 13km.conus.nam.grib2.conus -l select=1:ncpus=12:ompthreads=1:mem=15GB $PWD/13km.conus.nam.grib2.sh) + +#----------------------------------------------------------------------------- +# Initialize CONUS 13-KM USING RAP GRIB2 file WITH GSD PHYSICS . +#----------------------------------------------------------------------------- + +LOG_FILE=consistency.log13 +export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" +TEST13=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ + -N 13km.conus.rap.grib2.conus -l select=1:ncpus=12:ompthreads=1:mem=15GB $PWD/13km.conus.rap.grib2.sh) + +#----------------------------------------------------------------------------- +# Initialize CONUS 13-KM NA USING NCEI GFS GRIB2 file WITH GFS PHYSICS . +#----------------------------------------------------------------------------- + +LOG_FILE=consistency.log14 +export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" +TEST14=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ + -N 13km.na.gfs.ncei.grib2.conus -l select=1:ncpus=12:ompthreads=1:mem=15GB $PWD/13km.na.gfs.ncei.grib2.sh) + +#----------------------------------------------------------------------------- +# Initialize C96 WAM IC using FV3 gaussian netcdf files. +#----------------------------------------------------------------------------- + +LOG_FILE=consistency.log15 +export APRUN="mpiexec -n 12 -ppn 12 --cpu-bind core" +TEST15=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ + -N c96.fv3.netcdf2wam -l select=1:ncpus=12:ompthreads=1:mem=75GB $PWD/c96.fv3.netcdf2wam.sh) + +#----------------------------------------------------------------------------- +# Initialize CONUS 25-KM USING GFS PGRIB2+BGRIB2 files. +#----------------------------------------------------------------------------- + +LOG_FILE=consistency.log16 +export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" +TEST16=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ + -N 25km.conus.gfs.pbgrib2.conus -l select=1:ncpus=12:ompthreads=1:mem=15GB $PWD/25km.conus.gfs.pbgrib2.sh) + +#----------------------------------------------------------------------------- +# Create summary log. +#----------------------------------------------------------------------------- + +LOG_FILE=consistency.log +qsub -V -o ${LOG_FILE} -e ${LOG_FILE} -q $QUEUE -A $PROJECT_CODE -l walltime=00:01:00 \ + -N chgres_summary -l select=1:ncpus=2:mem=100MB \ + -W depend=afterok:$TEST1:$TEST2:$TEST3:$TEST4:$TEST5:$TEST6:$TEST7:$TEST8:$TEST9:$TEST10:$TEST11:$TEST12:$TEST13:$TEST14:$TEST15:$TEST16 << EOF +#!/bin/bash +cd ${this_dir} +grep -a '<<<' ${LOG_FILE}?? > $SUM_FILE +EOF + +exit 0 From bafbef374e2b9053325f4557a61c106b59228cfa Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Tue, 21 Sep 2021 12:22:41 +0000 Subject: [PATCH 14/57] Begin work on new driver script for gdas_init utility. Fixes #559. --- util/gdas_init/driver.wcoss2.sh | 215 ++++++++++++++++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100755 util/gdas_init/driver.wcoss2.sh diff --git a/util/gdas_init/driver.wcoss2.sh b/util/gdas_init/driver.wcoss2.sh new file mode 100755 index 000000000..ddcbe970d --- /dev/null +++ b/util/gdas_init/driver.wcoss2.sh @@ -0,0 +1,215 @@ +#!/bin/bash + +#--------------------------------------------------------------------- +# Driver script for running on WCOSS2. +# +# Edit the 'config' file before running. +#--------------------------------------------------------------------- + +set -x + +compiler=${compiler:-"intel"} +source ../../sorc/machine-setup.sh > /dev/null 2>&1 +module use ../../modulefiles +module load build.$target.$compiler +module list + +# Needed for NDATE utility +module load prod_util/2.0.8 + +PROJECT_CODE=GFS-DEV +QUEUE=dev + +source config + +this_dir=$PWD + +if [ $EXTRACT_DATA == yes ]; then + + rm -fr $EXTRACT_DIR + mkdir -p $EXTRACT_DIR + + QUEUE=dev_transfer + MEM=6000M + WALLT="02:00:00" + + case $gfs_ver in + v12 | v13) + DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \ + -o log.data.${CDUMP} -e log.data.${CDUMP} ./get_pre-v14.data.sh ${CDUMP}) + DEPEND="-d afterok:$DATAH" + if [ "$CDUMP" = "gdas" ] ; then + DATA1=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_enkf \ + -o log.data.enkf -e log.data.enkf ./get_pre-v14.data.sh enkf) + DEPEND="-d afterok:$DATAH:$DATA1" + fi + ;; + v14) + DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \ + -o log.data.${CDUMP} -e log.data.${CDUMP} ./get_v14.data.sh ${CDUMP}) + DEPEND="-d afterok:$DATAH" + if [ "$CDUMP" = "gdas" ] ; then + DATA1=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_enkf \ + -o log.data.enkf -e log.data.enkf ./get_v14.data.sh enkf) + DEPEND="-d afterok:$DATAH:$DATA1" + fi + ;; + v15) + if [ "$CDUMP" = "gfs" ] ; then + DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \ + -o log.data.${CDUMP} -e log.data.${CDUMP} ./get_v15.data.sh ${CDUMP}) + DEPEND="-d afterok:$DATAH" + else + DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \ + -o log.data.${CDUMP} -e log.data.${CDUMP} ./get_v15.data.sh ${CDUMP}) + DATA1=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp1 \ + -o log.data.grp1 -e log.data.grp1 ./get_v15.data.sh grp1) + DATA2=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp2 \ + -o log.data.grp2 -e log.data.grp2 ./get_v15.data.sh grp2) + DATA3=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp3 \ + -o log.data.grp3 -e log.data.grp3 ./get_v15.data.sh grp3) + DATA4=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp4 \ + -o log.data.grp4 -e log.data.grp4 ./get_v15.data.sh grp4) + DATA5=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp5 \ + -o log.data.grp5 -e log.data.grp5 ./get_v15.data.sh grp5) + DATA6=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp6 \ + -o log.data.grp6 -e log.data.grp6 ./get_v15.data.sh grp6) + DATA7=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp7 \ + -o log.data.grp7 -e log.data.grp7 ./get_v15.data.sh grp7) + DATA8=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp8 \ + -o log.data.grp8 -e log.data.grp8 ./get_v15.data.sh grp8) + DEPEND="-d afterok:$DATAH:$DATA1:$DATA2:$DATA3:$DATA4:$DATA5:$DATA6:$DATA7:$DATA8" + fi + ;; + v16retro) + DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_v16retro \ + -o log.data.v16retro -e log.data.v16retro ./get_v16retro.data.sh ${CDUMP}) + DEPEND="-d afterok:$DATAH" + ;; + v16) + DATAH=$(qsub -V -o log.data.${CDUMP} -e log.data.${CDUMP} -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ + -N get_${CDUMP} -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh ${CDUMP}) + DEPEND="-W depend=afterok:$DATAH" + if [ "$CDUMP" = "gdas" ] ; then + DATA1=$(qsub -V -o log.data.grp1 -e log.data.grp1 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ + -N get_grp1 -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh grp1) + DATA2=$(qsub -V -o log.data.grp2 -e log.data.grp2 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ + -N get_grp2 -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh grp2) + DATA3=$(qsub -V -o log.data.grp3 -e log.data.grp3 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ + -N get_grp3 -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh grp3) + DATA4=$(qsub -V -o log.data.grp4 -e log.data.grp4 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ + -N get_grp4 -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh grp4) + DATA5=$(qsub -V -o log.data.grp5 -e log.data.grp5 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ + -N get_grp5 -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh grp5) + DATA6=$(qsub -V -o log.data.grp6 -e log.data.grp6 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ + -N get_grp5 -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh grp6) + DATA7=$(qsub -V -o log.data.grp7 -e log.data.grp7 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ + -N get_grp7 -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh grp7) + DATA8=$(qsub -V -o log.data.grp8 -e log.data.grp8 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ + -N get_grp8 -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh grp8) + DEPEND="-W depend=afterok:$DATAH:$DATA1:$DATA2:$DATA3:$DATA4:$DATA5:$DATA6:$DATA7:$DATA8" + fi + ;; + esac + +else # do not extract data. + + DEPEND=' ' + +fi # extract data? + +if [ $RUN_CHGRES == yes ]; then + + export APRUN=srun + NODES=3 + WALLT="0:15:00" + export OMP_NUM_THREADS=1 + if [ $CRES_HIRES == 'C768' ] ; then + NODES=5 + elif [ $CRES_HIRES == 'C1152' ] ; then + NODES=8 + WALLT="0:20:00" + fi + case $gfs_ver in + v12 | v13) + export OMP_NUM_THREADS=4 + export OMP_STACKSIZE=1024M + sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} --cpus-per-task=$OMP_NUM_THREADS \ + -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ + -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_pre-v14.chgres.sh ${CDUMP} + ;; + v14) + sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ + -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v14.chgres.sh ${CDUMP} + ;; + v15) + if [ "$CDUMP" = "gdas" ]; then + sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ + -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v15.chgres.sh ${CDUMP} + else + sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ + -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v15.chgres.gfs.sh + fi + ;; + v16retro) + if [ "$CDUMP" = "gdas" ] ; then + sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ + -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v16retro.chgres.sh hires + else + sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ + -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v16.chgres.sh ${CDUMP} + fi + ;; + v16) + sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ + -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v16.chgres.sh ${CDUMP} + ;; + esac + + if [ "$CDUMP" = "gdas" ]; then + + WALLT="0:15:00" + + if [ "$gfs_ver" = "v16retro" ]; then + + sbatch --parsable --ntasks-per-node=12 --nodes=1 -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_enkf \ + -o log.enkf -e log.enkf ${DEPEND} run_v16retro.chgres.sh enkf + + else + + MEMBER=1 + while [ $MEMBER -le 80 ]; do + if [ $MEMBER -lt 10 ]; then + MEMBER_CH="00${MEMBER}" + else + MEMBER_CH="0${MEMBER}" + fi + case $gfs_ver in + v12 | v13) + export OMP_NUM_THREADS=2 + export OMP_STACKSIZE=1024M + sbatch --parsable --ntasks-per-node=12 --nodes=1 --cpus-per-task=$OMP_NUM_THREADS \ + -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \ + -o log.${MEMBER_CH} -e log.${MEMBER_CH} ${DEPEND} run_pre-v14.chgres.sh ${MEMBER_CH} + ;; + v14) + sbatch --parsable --ntasks-per-node=12 --nodes=1 -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \ + -o log.${MEMBER_CH} -e log.${MEMBER_CH} ${DEPEND} run_v14.chgres.sh ${MEMBER_CH} + ;; + v15) + sbatch --parsable --ntasks-per-node=12 --nodes=1 -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \ + -o log.${MEMBER_CH} -e log.${MEMBER_CH} ${DEPEND} run_v15.chgres.sh ${MEMBER_CH} + ;; + v16) + sbatch --parsable --ntasks-per-node=12 --nodes=1 -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \ + -o log.${MEMBER_CH} -e log.${MEMBER_CH} ${DEPEND} run_v16.chgres.sh ${MEMBER_CH} + ;; + esac + MEMBER=$(( $MEMBER + 1 )) + done + + fi # v16 retro? + + fi # which CDUMP? + +fi # run chgres? From 193903120e0612139460c1961ba3fc54a9f13278 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Tue, 21 Sep 2021 18:08:37 +0000 Subject: [PATCH 15/57] Complete mods for data pull scripts. Start mods for chgres scripts. Fixes #559. --- util/gdas_init/driver.wcoss2.sh | 109 ++++++++++++++++---------------- 1 file changed, 56 insertions(+), 53 deletions(-) diff --git a/util/gdas_init/driver.wcoss2.sh b/util/gdas_init/driver.wcoss2.sh index ddcbe970d..d1fc4e623 100755 --- a/util/gdas_init/driver.wcoss2.sh +++ b/util/gdas_init/driver.wcoss2.sh @@ -18,7 +18,6 @@ module list module load prod_util/2.0.8 PROJECT_CODE=GFS-DEV -QUEUE=dev source config @@ -30,83 +29,83 @@ if [ $EXTRACT_DATA == yes ]; then mkdir -p $EXTRACT_DIR QUEUE=dev_transfer - MEM=6000M + MEM=500MB WALLT="02:00:00" case $gfs_ver in v12 | v13) - DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \ - -o log.data.${CDUMP} -e log.data.${CDUMP} ./get_pre-v14.data.sh ${CDUMP}) - DEPEND="-d afterok:$DATAH" + DATAH=$(qsub -V -o log.data.${CDUMP} -e log.data.${CDUMP} -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ + -N get_${CDUMP} -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_pre-v14.data.sh ${CDUMP}) + DEPEND="-W depend=afterok:$DATAH" if [ "$CDUMP" = "gdas" ] ; then - DATA1=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_enkf \ - -o log.data.enkf -e log.data.enkf ./get_pre-v14.data.sh enkf) - DEPEND="-d afterok:$DATAH:$DATA1" + DATA1=$(qsub -V -o log.data.enkf -e log.data.enkf -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ + -N get_enkf -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_pre-v14.data.sh enkf) + DEPEND="-W depend=afterok:$DATAH:$DATA1" fi ;; v14) - DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \ - -o log.data.${CDUMP} -e log.data.${CDUMP} ./get_v14.data.sh ${CDUMP}) - DEPEND="-d afterok:$DATAH" + DATAH=$(qsub -V -o log.data.${CDUMP} -e log.data.${CDUMP} -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ + -N get_${CDUMP} -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v14.data.sh ${CDUMP}) + DEPEND="-W depend=afterok:$DATAH" if [ "$CDUMP" = "gdas" ] ; then - DATA1=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_enkf \ - -o log.data.enkf -e log.data.enkf ./get_v14.data.sh enkf) - DEPEND="-d afterok:$DATAH:$DATA1" + DATA1=$(qsub -V -o log.data.enkf -e log.data.enkf -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ + -N get_enkf -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v14.data.sh enkf) + DEPEND="-W depend=afterok:$DATAH:$DATA1" fi ;; v15) if [ "$CDUMP" = "gfs" ] ; then - DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \ - -o log.data.${CDUMP} -e log.data.${CDUMP} ./get_v15.data.sh ${CDUMP}) - DEPEND="-d afterok:$DATAH" + DATAH=$(qsub -V -o log.data.${CDUMP} -e log.data.${CDUMP} -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ + -N get_${CDUMP} -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh ${CDUMP}) + DEPEND="-W depend=afterok:$DATAH" else - DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \ - -o log.data.${CDUMP} -e log.data.${CDUMP} ./get_v15.data.sh ${CDUMP}) - DATA1=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp1 \ - -o log.data.grp1 -e log.data.grp1 ./get_v15.data.sh grp1) - DATA2=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp2 \ - -o log.data.grp2 -e log.data.grp2 ./get_v15.data.sh grp2) - DATA3=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp3 \ - -o log.data.grp3 -e log.data.grp3 ./get_v15.data.sh grp3) - DATA4=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp4 \ - -o log.data.grp4 -e log.data.grp4 ./get_v15.data.sh grp4) - DATA5=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp5 \ - -o log.data.grp5 -e log.data.grp5 ./get_v15.data.sh grp5) - DATA6=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp6 \ - -o log.data.grp6 -e log.data.grp6 ./get_v15.data.sh grp6) - DATA7=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp7 \ - -o log.data.grp7 -e log.data.grp7 ./get_v15.data.sh grp7) - DATA8=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp8 \ - -o log.data.grp8 -e log.data.grp8 ./get_v15.data.sh grp8) - DEPEND="-d afterok:$DATAH:$DATA1:$DATA2:$DATA3:$DATA4:$DATA5:$DATA6:$DATA7:$DATA8" + DATAH=$(qsub -V -o log.data.${CDUMP} -e log.data.${CDUMP} -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ + -N get_${CDUMP} -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh ${CDUMP}) + DATA1=$(qsub -V -o log.data.grp1 -e log.data.grp1 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ + -N get_grp1 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh grp1) + DATA2=$(qsub -V -o log.data.grp2 -e log.data.grp2 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ + -N get_grp2 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh grp2) + DATA3=$(qsub -V -o log.data.grp3 -e log.data.grp3 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ + -N get_grp3 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh grp3) + DATA4=$(qsub -V -o log.data.grp4 -e log.data.grp4 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ + -N get_grp4 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh grp4) + DATA5=$(qsub -V -o log.data.grp5 -e log.data.grp5 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ + -N get_grp5 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh grp5) + DATA6=$(qsub -V -o log.data.grp6 -e log.data.grp6 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ + -N get_grp6 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh grp6) + DATA7=$(qsub -V -o log.data.grp7 -e log.data.grp7 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ + -N get_grp7 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh grp7) + DATA8=$(qsub -V -o log.data.grp8 -e log.data.grp8 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ + -N get_grp8 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh grp8) + DEPEND="-W depend=afterok:$DATAH:$DATA1:$DATA2:$DATA3:$DATA4:$DATA5:$DATA6:$DATA7:$DATA8" fi ;; v16retro) - DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_v16retro \ - -o log.data.v16retro -e log.data.v16retro ./get_v16retro.data.sh ${CDUMP}) - DEPEND="-d afterok:$DATAH" + DATAH=$(qsub -V -o log.data.v16retro -e log.data.v16retro -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ + -N get_v16retro -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16retro.data.sh ${CDUMP}) + DEPEND="-W depend=afterok:$DATAH" ;; v16) DATAH=$(qsub -V -o log.data.${CDUMP} -e log.data.${CDUMP} -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_${CDUMP} -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh ${CDUMP}) + -N get_${CDUMP} -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh ${CDUMP}) DEPEND="-W depend=afterok:$DATAH" if [ "$CDUMP" = "gdas" ] ; then DATA1=$(qsub -V -o log.data.grp1 -e log.data.grp1 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_grp1 -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh grp1) + -N get_grp1 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh grp1) DATA2=$(qsub -V -o log.data.grp2 -e log.data.grp2 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_grp2 -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh grp2) + -N get_grp2 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh grp2) DATA3=$(qsub -V -o log.data.grp3 -e log.data.grp3 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_grp3 -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh grp3) + -N get_grp3 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh grp3) DATA4=$(qsub -V -o log.data.grp4 -e log.data.grp4 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_grp4 -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh grp4) + -N get_grp4 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh grp4) DATA5=$(qsub -V -o log.data.grp5 -e log.data.grp5 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_grp5 -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh grp5) + -N get_grp5 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh grp5) DATA6=$(qsub -V -o log.data.grp6 -e log.data.grp6 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_grp5 -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh grp6) + -N get_grp5 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh grp6) DATA7=$(qsub -V -o log.data.grp7 -e log.data.grp7 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_grp7 -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh grp7) + -N get_grp7 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh grp7) DATA8=$(qsub -V -o log.data.grp8 -e log.data.grp8 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_grp8 -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh grp8) + -N get_grp8 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh grp8) DEPEND="-W depend=afterok:$DATAH:$DATA1:$DATA2:$DATA3:$DATA4:$DATA5:$DATA6:$DATA7:$DATA8" fi ;; @@ -120,16 +119,20 @@ fi # extract data? if [ $RUN_CHGRES == yes ]; then - export APRUN=srun - NODES=3 + QUEUE=dev + NODES=1 + TASKS_PER_NODE=12 WALLT="0:15:00" - export OMP_NUM_THREADS=1 + MEM=75GB if [ $CRES_HIRES == 'C768' ] ; then NODES=5 elif [ $CRES_HIRES == 'C1152' ] ; then NODES=8 WALLT="0:20:00" fi + (( NCPUS = TASKS_PER_NODE * 2 )) + (( TASKS = NODES * TASKS_PER_NODE)) + export APRUN="mpiexec -n $TASKS -ppn $TASKS_PER_NODE --cpu-bind core" case $gfs_ver in v12 | v13) export OMP_NUM_THREADS=4 @@ -161,8 +164,8 @@ if [ $RUN_CHGRES == yes ]; then fi ;; v16) - sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ - -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v16.chgres.sh ${CDUMP} + qsub -V -l select=${NODES}:ncpus=${NCPUS}:ompthreads=1:mem=${MEM} -l walltime=$WALLT -A $PROJECT_CODE -q $QUEUE \ + -N chgres_${CDUMP} -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} -- ${this_dir}/run_v16.chgres.sh ${CDUMP} ;; esac From 7b7cdf0874f435a87d3ddfc50a3afb22158ff40f Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 23 Sep 2021 21:07:35 +0000 Subject: [PATCH 16/57] Add qsub commands to remainder of ./util/gdas_init/driver.wcoss2.sh script. Fixes #559 --- util/gdas_init/driver.wcoss2.sh | 79 ++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 31 deletions(-) diff --git a/util/gdas_init/driver.wcoss2.sh b/util/gdas_init/driver.wcoss2.sh index d1fc4e623..96b99d2c8 100755 --- a/util/gdas_init/driver.wcoss2.sh +++ b/util/gdas_init/driver.wcoss2.sh @@ -121,46 +121,50 @@ if [ $RUN_CHGRES == yes ]; then QUEUE=dev NODES=1 - TASKS_PER_NODE=12 + TASKS_PER_NODE=24 WALLT="0:15:00" MEM=75GB if [ $CRES_HIRES == 'C768' ] ; then - NODES=5 + MEM=175GB elif [ $CRES_HIRES == 'C1152' ] ; then - NODES=8 + MEM=350GB + NODES=1 + TASKS_PER_NODE=48 WALLT="0:20:00" fi (( NCPUS = TASKS_PER_NODE * 2 )) - (( TASKS = NODES * TASKS_PER_NODE)) + (( TASKS = NODES * TASKS_PER_NODE )) export APRUN="mpiexec -n $TASKS -ppn $TASKS_PER_NODE --cpu-bind core" case $gfs_ver in v12 | v13) export OMP_NUM_THREADS=4 export OMP_STACKSIZE=1024M - sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} --cpus-per-task=$OMP_NUM_THREADS \ - -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ - -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_pre-v14.chgres.sh ${CDUMP} + export OMP_PLACES=cores + export APRUN="$APRUN --depth 2" + (( NCPUS = NCPUS * OMP_NUM_THREADS )) + qsub -V -l select=${NODES}:ncpus=${NCPUS}:ompthreads=${OMP_NUM_THREADS}:mem=${MEM} -l walltime=$WALLT -A $PROJECT_CODE -q $QUEUE \ + -N chgres_${CDUMP} -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} -- ${this_dir}/run_pre-v14.chgres.sh ${CDUMP} ;; v14) - sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ - -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v14.chgres.sh ${CDUMP} + qsub -V -l select=${NODES}:ncpus=${NCPUS}:ompthreads=1:mem=${MEM} -l walltime=$WALLT -A $PROJECT_CODE -q $QUEUE \ + -N chgres_${CDUMP} -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} -- ${this_dir}/run_v14.chgres.sh ${CDUMP} ;; v15) if [ "$CDUMP" = "gdas" ]; then - sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ - -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v15.chgres.sh ${CDUMP} + qsub -V -l select=${NODES}:ncpus=${NCPUS}:ompthreads=1:mem=${MEM} -l walltime=$WALLT -A $PROJECT_CODE -q $QUEUE \ + -N chgres_${CDUMP} -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} -- ${this_dir}/run_v15.chgres.sh ${CDUMP} else - sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ - -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v15.chgres.gfs.sh + qsub -V -l select=${NODES}:ncpus=${NCPUS}:ompthreads=1:mem=${MEM} -l walltime=$WALLT -A $PROJECT_CODE -q $QUEUE \ + -N chgres_${CDUMP} -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} ${this_dir}/run_v15.chgres.gfs.sh fi ;; v16retro) if [ "$CDUMP" = "gdas" ] ; then - sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ - -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v16retro.chgres.sh hires + qsub -V -l select=${NODES}:ncpus=${NCPUS}:ompthreads=1:mem=${MEM} -l walltime=$WALLT -A $PROJECT_CODE -q $QUEUE \ + -N chgres_${CDUMP} -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} -- ${this_dir}/run_v16retro.chgres.sh hires else - sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \ - -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v16.chgres.sh ${CDUMP} + qsub -V -l select=${NODES}:ncpus=${NCPUS}:ompthreads=1:mem=${MEM} -l walltime=$WALLT -A $PROJECT_CODE -q $QUEUE \ + -N chgres_${CDUMP} -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} -- ${this_dir}/run_v16.chgres.sh ${CDUMP} fi ;; v16) @@ -172,16 +176,32 @@ if [ $RUN_CHGRES == yes ]; then if [ "$CDUMP" = "gdas" ]; then WALLT="0:15:00" + MEM=75GB + NODES=1 + TASKS_PER_NODE=12 + (( NCPUS = TASKS_PER_NODE * 2 )) + (( TASKS = NODES * TASKS_PER_NODE)) + export APRUN="mpiexec -n $TASKS -ppn $TASKS_PER_NODE --cpu-bind core" if [ "$gfs_ver" = "v16retro" ]; then - sbatch --parsable --ntasks-per-node=12 --nodes=1 -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_enkf \ - -o log.enkf -e log.enkf ${DEPEND} run_v16retro.chgres.sh enkf + qsub -V -l select=${NODES}:ncpus=${NCPUS}:ompthreads=1:mem=${MEM} -l walltime=$WALLT -A $PROJECT_CODE -q $QUEUE \ + -N chgres_enkf -o log.enkf -e log.enkf ${DEPEND} -- ${this_dir}/run_v16retro.chgres.sh enkf else + case $gfs_ver in # use threads for v12/13 data. + v12 | v13) + export OMP_NUM_THREADS=2 + export OMP_STACKSIZE=1024M + export OMP_PLACES=cores + export APRUN="$APRUN --depth 2" + (( NCPUS = NCPUS * OMP_NUM_THREADS )) + ;; + esac + MEMBER=1 - while [ $MEMBER -le 80 ]; do + while [ $MEMBER -le 04 ]; do if [ $MEMBER -lt 10 ]; then MEMBER_CH="00${MEMBER}" else @@ -189,23 +209,20 @@ if [ $RUN_CHGRES == yes ]; then fi case $gfs_ver in v12 | v13) - export OMP_NUM_THREADS=2 - export OMP_STACKSIZE=1024M - sbatch --parsable --ntasks-per-node=12 --nodes=1 --cpus-per-task=$OMP_NUM_THREADS \ - -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \ - -o log.${MEMBER_CH} -e log.${MEMBER_CH} ${DEPEND} run_pre-v14.chgres.sh ${MEMBER_CH} + qsub -V -l select=${NODES}:ncpus=${NCPUS}:ompthreads=${OMP_NUM_THREADS}:mem=${MEM} -l walltime=$WALLT -A $PROJECT_CODE -q $QUEUE \ + -N chgres_${MEMBER_CH} -o log.${MEMBER_CH} -e log.${MEMBER_CH} ${DEPEND} -- ${this_dir}/run_pre-v14.chgres.sh ${MEMBER_CH} ;; v14) - sbatch --parsable --ntasks-per-node=12 --nodes=1 -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \ - -o log.${MEMBER_CH} -e log.${MEMBER_CH} ${DEPEND} run_v14.chgres.sh ${MEMBER_CH} + qsub -V -l select=${NODES}:ncpus=${NCPUS}:ompthreads=1:mem=${MEM} -l walltime=$WALLT -A $PROJECT_CODE -q $QUEUE \ + -N chgres_${MEMBER_CH} -o log.${MEMBER_CH} -e log.${MEMBER_CH} ${DEPEND} -- ${this_dir}/run_v14.chgres.sh ${MEMBER_CH} ;; v15) - sbatch --parsable --ntasks-per-node=12 --nodes=1 -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \ - -o log.${MEMBER_CH} -e log.${MEMBER_CH} ${DEPEND} run_v15.chgres.sh ${MEMBER_CH} + qsub -V -l select=${NODES}:ncpus=${NCPUS}:ompthreads=1:mem=${MEM} -l walltime=$WALLT -A $PROJECT_CODE -q $QUEUE \ + -N chgres_${MEMBER_CH} -o log.${MEMBER_CH} -e log.${MEMBER_CH} ${DEPEND} -- ${this_dir}/run_v15.chgres.sh ${MEMBER_CH} ;; v16) - sbatch --parsable --ntasks-per-node=12 --nodes=1 -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \ - -o log.${MEMBER_CH} -e log.${MEMBER_CH} ${DEPEND} run_v16.chgres.sh ${MEMBER_CH} + qsub -V -l select=${NODES}:ncpus=${NCPUS}:ompthreads=1:mem=${MEM} -l walltime=$WALLT -A $PROJECT_CODE -q $QUEUE \ + -N chgres_${MEMBER_CH} -o log.${MEMBER_CH} -e log.${MEMBER_CH} ${DEPEND} -- ${this_dir}/run_v16.chgres.sh ${MEMBER_CH} ;; esac MEMBER=$(( $MEMBER + 1 )) From 7b0d1f43a449e132783d8de4e3e2e4a1a4cf7f04 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Fri, 24 Sep 2021 18:49:32 +0000 Subject: [PATCH 17/57] Hardware threading was turned off on WCOSS2. Start adjusting the scripts accordingly. Fixes #559. --- reg_tests/ice_blend/driver.wcoss2.sh | 2 +- reg_tests/snow2mdl/driver.wcoss2.sh | 2 +- util/sfc_climo_gen/run.wcoss2.sh | 6 ++++-- util/vcoord_gen/run.wcoss2.sh | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/reg_tests/ice_blend/driver.wcoss2.sh b/reg_tests/ice_blend/driver.wcoss2.sh index af60196a8..c58062ae5 100755 --- a/reg_tests/ice_blend/driver.wcoss2.sh +++ b/reg_tests/ice_blend/driver.wcoss2.sh @@ -24,7 +24,7 @@ #PBS -N iceb_regt #PBS -q debug #PBS -A GFS-DEV -#PBS -l select=1:ncpus=2:mem=2500MB +#PBS -l select=1:ncpus=1:mem=2500MB cd $PBS_O_WORKDIR diff --git a/reg_tests/snow2mdl/driver.wcoss2.sh b/reg_tests/snow2mdl/driver.wcoss2.sh index 6d43d558f..cc792ee34 100755 --- a/reg_tests/snow2mdl/driver.wcoss2.sh +++ b/reg_tests/snow2mdl/driver.wcoss2.sh @@ -24,7 +24,7 @@ #PBS -N s2m_regt #PBS -q debug #PBS -A GFS-DEV -#PBS -l select=1:ncpus=2:mem=2500MB +#PBS -l select=1:ncpus=1:mem=2500MB cd $PBS_O_WORKDIR diff --git a/util/sfc_climo_gen/run.wcoss2.sh b/util/sfc_climo_gen/run.wcoss2.sh index 39232e52c..63e9c692f 100755 --- a/util/sfc_climo_gen/run.wcoss2.sh +++ b/util/sfc_climo_gen/run.wcoss2.sh @@ -13,10 +13,13 @@ #PBS -A GFS-DEV #PBS -N grid_fv3 #PBS -l walltime=00:10:00 -#PBS -l select=1:ncpus=48 +#PBS -l select=1:ncpus=24:mem=75GB set -x +# Adjust according to the PBS -l statement. +export APRUN_SFC="mpiexec -n 24 -ppn 24 -cpu-bind core" + export BASE_DIR=$PBS_O_WORKDIR/../.. source ${BASE_DIR}/sorc/machine-setup.sh > /dev/null 2>&1 @@ -72,7 +75,6 @@ if [[ $GRIDTYPE = "regional" ]]; then fi export input_sfc_climo_dir=${BASE_DIR}/fix/fix_sfc_climo -export APRUN_SFC="mpiexec -n 24 -ppn 24" ulimit -a ulimit -s unlimited diff --git a/util/vcoord_gen/run.wcoss2.sh b/util/vcoord_gen/run.wcoss2.sh index 7f2e096fc..7a53c8533 100755 --- a/util/vcoord_gen/run.wcoss2.sh +++ b/util/vcoord_gen/run.wcoss2.sh @@ -6,7 +6,7 @@ #PBS -N vcoord #PBS -q debug #PBS -A GFS-DEV -#PBS -l select=1:ncpus=2:mem=1MB +#PBS -l select=1:ncpus=1:mem=1MB #------------------------------------------------------------------------------- # From b49c1382dec3b0293f4fa480300a111ef8604a27 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Fri, 24 Sep 2021 19:25:19 +0000 Subject: [PATCH 18/57] Update more scripts now that hardware threading is turned off. Fixes #559. --- reg_tests/chgres_cube/driver.wcoss2.sh | 32 ++++++++++++------------- reg_tests/global_cycle/driver.wcoss2.sh | 10 ++++---- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/reg_tests/chgres_cube/driver.wcoss2.sh b/reg_tests/chgres_cube/driver.wcoss2.sh index e09955094..e7eed0333 100755 --- a/reg_tests/chgres_cube/driver.wcoss2.sh +++ b/reg_tests/chgres_cube/driver.wcoss2.sh @@ -65,7 +65,7 @@ this_dir=$PWD LOG_FILE=consistency.log01 export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" TEST1=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N c96.fv3.restart -l select=1:ncpus=12:ompthreads=1:mem=10GB $PWD/c96.fv3.restart.sh) + -N c96.fv3.restart -l select=1:ncpus=6:ompthreads=1:mem=10GB $PWD/c96.fv3.restart.sh) #----------------------------------------------------------------------------- # Initialize C192 using FV3 tiled history files. @@ -74,7 +74,7 @@ TEST1=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime LOG_FILE=consistency.log02 export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" TEST2=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N c192.fv3.history -l select=1:ncpus=12:ompthreads=1:mem=10GB $PWD/c192.fv3.history.sh) + -N c192.fv3.history -l select=1:ncpus=6:ompthreads=1:mem=10GB $PWD/c192.fv3.history.sh) #----------------------------------------------------------------------------- # Initialize C96 using FV3 gaussian nemsio files. @@ -83,7 +83,7 @@ TEST2=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime LOG_FILE=consistency.log03 export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" TEST3=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N c96.fv3.nemsio -l select=1:ncpus=12:ompthreads=1:mem=45GB $PWD/c96.fv3.nemsio.sh) + -N c96.fv3.nemsio -l select=1:ncpus=6:ompthreads=1:mem=45GB $PWD/c96.fv3.nemsio.sh) #----------------------------------------------------------------------------- # Initialize C96 using spectral GFS sigio/sfcio files. @@ -93,7 +93,7 @@ LOG_FILE=consistency.log04 export OMP_PLACES=cores export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core --depth 4" TEST4=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:10:00 \ - -N c96.gfs.sigio -l select=1:ncpus=48:ompthreads=4:mem=45GB $PWD/c96.gfs.sigio.sh) + -N c96.gfs.sigio -l select=1:ncpus=24:ompthreads=4:mem=45GB $PWD/c96.gfs.sigio.sh) #----------------------------------------------------------------------------- # Initialize C96 using spectral GFS gaussian nemsio files. @@ -102,7 +102,7 @@ TEST4=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime LOG_FILE=consistency.log05 export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" TEST5=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N c96.gfs.nemsio -l select=1:ncpus=12:ompthreads=1:mem=35GB $PWD/c96.gfs.nemsio.sh) + -N c96.gfs.nemsio -l select=1:ncpus=6:ompthreads=1:mem=35GB $PWD/c96.gfs.nemsio.sh) #----------------------------------------------------------------------------- # Initialize regional C96 using FV3 gaussian nemsio files. @@ -111,7 +111,7 @@ TEST5=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime LOG_FILE=consistency.log06 export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" TEST6=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N c96.regional -l select=1:ncpus=12:ompthreads=1:mem=35GB $PWD/c96.regional.sh) + -N c96.regional -l select=1:ncpus=6:ompthreads=1:mem=35GB $PWD/c96.regional.sh) #----------------------------------------------------------------------------- # Initialize C96 using FV3 gaussian netcdf files. @@ -120,7 +120,7 @@ TEST6=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime LOG_FILE=consistency.log07 export APRUN="mpiexec -n 12 -ppn 12 --cpu-bind core" TEST7=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N c96.fv3.netcdf -l select=1:ncpus=24:ompthreads=1:mem=80GB $PWD/c96.fv3.netcdf.sh) + -N c96.fv3.netcdf -l select=1:ncpus=12:ompthreads=1:mem=80GB $PWD/c96.fv3.netcdf.sh) #----------------------------------------------------------------------------- # Initialize global C192 using GFS GRIB2 files. @@ -129,7 +129,7 @@ TEST7=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime LOG_FILE=consistency.log08 export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" TEST8=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N c192.gfs.grib2 -l select=1:ncpus=12:ompthreads=1:mem=15GB $PWD/c192.gfs.grib2.sh) + -N c192.gfs.grib2 -l select=1:ncpus=6:ompthreads=1:mem=15GB $PWD/c192.gfs.grib2.sh) #----------------------------------------------------------------------------- # Initialize CONUS 25-KM USING GFS GRIB2 files. @@ -138,7 +138,7 @@ TEST8=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime LOG_FILE=consistency.log09 export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" TEST9=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N 25km.conus.gfs.grib2.conus -l select=1:ncpus=12:ompthreads=1:mem=15GB $PWD/25km.conus.gfs.grib2.sh) + -N 25km.conus.gfs.grib2.conus -l select=1:ncpus=6:ompthreads=1:mem=15GB $PWD/25km.conus.gfs.grib2.sh) #----------------------------------------------------------------------------- # Initialize CONUS 3-KM USING HRRR GRIB2 file WITH GFS PHYSICS. @@ -147,7 +147,7 @@ TEST9=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime LOG_FILE=consistency.log10 export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" TEST10=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N 3km.conus.hrrr.gfssdf.grib2.conus -l select=1:ncpus=12:ompthreads=1:mem=40GB $PWD/3km.conus.hrrr.gfssdf.grib2.sh) + -N 3km.conus.hrrr.gfssdf.grib2.conus -l select=1:ncpus=6:ompthreads=1:mem=40GB $PWD/3km.conus.hrrr.gfssdf.grib2.sh) #----------------------------------------------------------------------------- # Initialize CONUS 3-KM USING HRRR GRIB2 file WITH GSD PHYSICS AND SFC VARS FROM FILE. @@ -156,7 +156,7 @@ TEST10=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltim LOG_FILE=consistency.log11 export APRUN="mpiexec -n 12 -ppn 12 --cpu-bind core" TEST11=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N 3km.conus.hrrr.newsfc.grib2.conus -l select=1:ncpus=24:ompthreads=1:mem=50GB $PWD/3km.conus.hrrr.newsfc.grib2.sh) + -N 3km.conus.hrrr.newsfc.grib2.conus -l select=1:ncpus=12:ompthreads=1:mem=50GB $PWD/3km.conus.hrrr.newsfc.grib2.sh) #----------------------------------------------------------------------------- # Initialize CONUS 13-KM USING NAM GRIB2 file WITH GFS PHYSICS . @@ -165,7 +165,7 @@ TEST11=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltim LOG_FILE=consistency.log12 export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" TEST12=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N 13km.conus.nam.grib2.conus -l select=1:ncpus=12:ompthreads=1:mem=15GB $PWD/13km.conus.nam.grib2.sh) + -N 13km.conus.nam.grib2.conus -l select=1:ncpus=6:ompthreads=1:mem=15GB $PWD/13km.conus.nam.grib2.sh) #----------------------------------------------------------------------------- # Initialize CONUS 13-KM USING RAP GRIB2 file WITH GSD PHYSICS . @@ -174,7 +174,7 @@ TEST12=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltim LOG_FILE=consistency.log13 export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" TEST13=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N 13km.conus.rap.grib2.conus -l select=1:ncpus=12:ompthreads=1:mem=15GB $PWD/13km.conus.rap.grib2.sh) + -N 13km.conus.rap.grib2.conus -l select=1:ncpus=6:ompthreads=1:mem=15GB $PWD/13km.conus.rap.grib2.sh) #----------------------------------------------------------------------------- # Initialize CONUS 13-KM NA USING NCEI GFS GRIB2 file WITH GFS PHYSICS . @@ -183,7 +183,7 @@ TEST13=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltim LOG_FILE=consistency.log14 export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" TEST14=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N 13km.na.gfs.ncei.grib2.conus -l select=1:ncpus=12:ompthreads=1:mem=15GB $PWD/13km.na.gfs.ncei.grib2.sh) + -N 13km.na.gfs.ncei.grib2.conus -l select=1:ncpus=6:ompthreads=1:mem=15GB $PWD/13km.na.gfs.ncei.grib2.sh) #----------------------------------------------------------------------------- # Initialize C96 WAM IC using FV3 gaussian netcdf files. @@ -201,7 +201,7 @@ TEST15=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltim LOG_FILE=consistency.log16 export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" TEST16=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N 25km.conus.gfs.pbgrib2.conus -l select=1:ncpus=12:ompthreads=1:mem=15GB $PWD/25km.conus.gfs.pbgrib2.sh) + -N 25km.conus.gfs.pbgrib2.conus -l select=1:ncpus=6:ompthreads=1:mem=15GB $PWD/25km.conus.gfs.pbgrib2.sh) #----------------------------------------------------------------------------- # Create summary log. @@ -209,7 +209,7 @@ TEST16=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltim LOG_FILE=consistency.log qsub -V -o ${LOG_FILE} -e ${LOG_FILE} -q $QUEUE -A $PROJECT_CODE -l walltime=00:01:00 \ - -N chgres_summary -l select=1:ncpus=2:mem=100MB \ + -N chgres_summary -l select=1:ncpus=1:mem=100MB \ -W depend=afterok:$TEST1:$TEST2:$TEST3:$TEST4:$TEST5:$TEST6:$TEST7:$TEST8:$TEST9:$TEST10:$TEST11:$TEST12:$TEST13:$TEST14:$TEST15:$TEST16 << EOF #!/bin/bash cd ${this_dir} diff --git a/reg_tests/global_cycle/driver.wcoss2.sh b/reg_tests/global_cycle/driver.wcoss2.sh index 445ab3cc4..6dba4e96f 100755 --- a/reg_tests/global_cycle/driver.wcoss2.sh +++ b/reg_tests/global_cycle/driver.wcoss2.sh @@ -43,7 +43,7 @@ export HOMEreg=/lfs/h2/emc/eib/noscrub/George.Gayno/ufs_utils.git/reg_tests/glob export OMP_NUM_THREADS_CY=2 export OMP_PLACES=cores -export APRUNCY="mpiexec -n 6 -ppn 6 --cpu-bind core --depth 2" +export APRUNCY="mpiexec -n 6 -ppn 6 --cpu-bind core --depth ${OMP_NUM_THREADS_CY}" export NWPROD=$PWD/../.. @@ -55,20 +55,20 @@ rm -f ${LOG_FILE}* export DATA="${DATA_DIR}/test1" export COMOUT=$DATA TEST1=$(qsub -V -o ${LOG_FILE}01 -e ${LOG_FILE}01 -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N c768.fv3gfs -l select=1:ncpus=24:mem=8GB $PWD/C768.fv3gfs.sh) + -N c768.fv3gfs -l select=1:ncpus=12:mem=8GB $PWD/C768.fv3gfs.sh) export DATA="${DATA_DIR}/test2" export COMOUT=$DATA TEST2=$(qsub -V -o ${LOG_FILE}02 -e ${LOG_FILE}02 -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N c768.lndincsoil -l select=1:ncpus=24:mem=1GB $PWD/C768.lndincsoil.sh) + -N c768.lndincsoil -l select=1:ncpus=12:mem=1GB $PWD/C768.lndincsoil.sh) export DATA="${DATA_DIR}/test3" export COMOUT=$DATA TEST3=$(qsub -V -o ${LOG_FILE}03 -e ${LOG_FILE}03 -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N c768.lndincsnow -l select=1:ncpus=24:mem=1GB $PWD/C768.lndincsnow.sh) + -N c768.lndincsnow -l select=1:ncpus=12:mem=1GB $PWD/C768.lndincsnow.sh) qsub -V -o ${LOG_FILE} -e ${LOG_FILE} -q $QUEUE -A $PROJECT_CODE -l walltime=00:01:00 \ - -N cycle_summary -l select=1:ncpus=2:mem=100MB -W depend=afterok:$TEST1:$TEST2:$TEST3 << EOF + -N cycle_summary -l select=1:ncpus=1:mem=100MB -W depend=afterok:$TEST1:$TEST2:$TEST3 << EOF #!/bin/bash cd $reg_dir grep -a '<<<' ${LOG_FILE}?? > summary.log From 5b502941888fef743755122ad874e8b32a8b2cb8 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Fri, 24 Sep 2021 20:25:16 +0000 Subject: [PATCH 19/57] Update more scripts for new hardware configuration. Fixes #559. --- driver_scripts/driver_grid.wcoss2.sh | 5 +++-- reg_tests/grid_gen/driver.wcoss2.sh | 19 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/driver_scripts/driver_grid.wcoss2.sh b/driver_scripts/driver_grid.wcoss2.sh index f0d853893..90eac38b3 100755 --- a/driver_scripts/driver_grid.wcoss2.sh +++ b/driver_scripts/driver_grid.wcoss2.sh @@ -6,7 +6,7 @@ #PBS -A GFS-DEV #PBS -l walltime=00:05:00 #PBS -N make_grid -#PBS -l select=1:ncpus=48 +#PBS -l select=1:ncpus=24:mem=100GB #----------------------------------------------------------------------- # Driver script to create a cubic-sphere based model grid on WCOSS2. @@ -141,8 +141,9 @@ export out_dir=/lfs/h2/emc/stmp/$LOGNAME/my_grids set -x export APRUN=time -export APRUN_SFC="mpiexec -n 24 -ppn 24" +export APRUN_SFC="mpiexec -n 24 -ppn 24 -cpu-bind core" export OMP_NUM_THREADS=24 # orog code worked best with 24 threads. +export OMP_PLACES=cores export OMP_STACKSIZE=2048m export machine=WCOSS2_CRAY diff --git a/reg_tests/grid_gen/driver.wcoss2.sh b/reg_tests/grid_gen/driver.wcoss2.sh index 67b3d6a3d..17f817f33 100755 --- a/reg_tests/grid_gen/driver.wcoss2.sh +++ b/reg_tests/grid_gen/driver.wcoss2.sh @@ -43,8 +43,10 @@ rm -f ${LOG_FILE} SUM_FILE=summary.log export home_dir=$PWD/../.. export APRUN=time -export APRUN_SFC="mpiexec -n 30 -ppn 30" +export APRUN_SFC="mpiexec -n 30 -ppn 30 -cpu-bind core" export OMP_STACKSIZE=2048m +export OMP_NUM_THREADS=30 # orog code uses threads +export OMP_PLACES=cores export machine=WCOSS2 export HOMEreg=/lfs/h2/emc/eib/noscrub/George.Gayno/ufs_utils.git/reg_tests/grid_gen/baseline_data this_dir=$PWD @@ -53,50 +55,47 @@ ulimit -a rm -fr $WORK_DIR -export OMP_NUM_THREADS=30 -export OMP_PLACES=cores - #----------------------------------------------------------------------------- # C96 uniform grid #----------------------------------------------------------------------------- TEST1=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:08:00 \ - -N c96.uniform -l select=1:ncpus=60:mem=40GB $PWD/c96.uniform.sh) + -N c96.uniform -l select=1:ncpus=30:mem=40GB $PWD/c96.uniform.sh) #----------------------------------------------------------------------------- # C96 uniform grid using viirs vegetation data. #----------------------------------------------------------------------------- TEST2=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:08:00 \ - -N c96.viirs.vegt -l select=1:ncpus=60:mem=40GB -W depend=afterok:$TEST1 $PWD/c96.viirs.vegt.sh) + -N c96.viirs.vegt -l select=1:ncpus=30:mem=40GB -W depend=afterok:$TEST1 $PWD/c96.viirs.vegt.sh) #----------------------------------------------------------------------------- # gfdl regional grid #----------------------------------------------------------------------------- TEST3=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:08:00 \ - -N gfdl.regional -l select=1:ncpus=60:mem=40GB -W depend=afterok:$TEST2 $PWD/gfdl.regional.sh) + -N gfdl.regional -l select=1:ncpus=30:mem=40GB -W depend=afterok:$TEST2 $PWD/gfdl.regional.sh) #----------------------------------------------------------------------------- # esg regional grid #----------------------------------------------------------------------------- TEST4=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:08:00 \ - -N esg.regional -l select=1:ncpus=60:mem=40GB -W depend=afterok:$TEST3 $PWD/esg.regional.sh) + -N esg.regional -l select=1:ncpus=30:mem=40GB -W depend=afterok:$TEST3 $PWD/esg.regional.sh) #----------------------------------------------------------------------------- # Regional GSL gravity wave drag test. #----------------------------------------------------------------------------- TEST5=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:08:00 \ - -N rsg.gsl.gwd -l select=1:ncpus=60:mem=40GB -W depend=afterok:$TEST4 $PWD/regional.gsl.gwd.sh) + -N rsg.gsl.gwd -l select=1:ncpus=30:mem=40GB -W depend=afterok:$TEST4 $PWD/regional.gsl.gwd.sh) #----------------------------------------------------------------------------- # Create summary log. #----------------------------------------------------------------------------- qsub -V -o ${LOG_FILE} -e ${LOG_FILE} -q $QUEUE -A $PROJECT_CODE -l walltime=00:01:00 \ - -N grid_summary -l select=1:ncpus=2:mem=100MB -W depend=afterok:$TEST5 << EOF + -N grid_summary -l select=1:ncpus=1:mem=100MB -W depend=afterok:$TEST5 << EOF #!/bin/bash cd ${this_dir} grep -a '<<<' $LOG_FILE > $SUM_FILE From 49675fd2c3df339b125c006a643263f63fb130a0 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Fri, 24 Sep 2021 21:16:24 +0000 Subject: [PATCH 20/57] Update gdas_init driver script for new hardware configuration. Fixes #559. --- util/gdas_init/driver.wcoss2.sh | 56 ++++++++++++++++----------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/util/gdas_init/driver.wcoss2.sh b/util/gdas_init/driver.wcoss2.sh index 96b99d2c8..899532daa 100755 --- a/util/gdas_init/driver.wcoss2.sh +++ b/util/gdas_init/driver.wcoss2.sh @@ -35,77 +35,77 @@ if [ $EXTRACT_DATA == yes ]; then case $gfs_ver in v12 | v13) DATAH=$(qsub -V -o log.data.${CDUMP} -e log.data.${CDUMP} -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_${CDUMP} -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_pre-v14.data.sh ${CDUMP}) + -N get_${CDUMP} -l select=1:ncpus=1:mem=$MEM -- ${this_dir}/get_pre-v14.data.sh ${CDUMP}) DEPEND="-W depend=afterok:$DATAH" if [ "$CDUMP" = "gdas" ] ; then DATA1=$(qsub -V -o log.data.enkf -e log.data.enkf -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_enkf -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_pre-v14.data.sh enkf) + -N get_enkf -l select=1:ncpus=1:mem=$MEM -- ${this_dir}/get_pre-v14.data.sh enkf) DEPEND="-W depend=afterok:$DATAH:$DATA1" fi ;; v14) DATAH=$(qsub -V -o log.data.${CDUMP} -e log.data.${CDUMP} -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_${CDUMP} -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v14.data.sh ${CDUMP}) + -N get_${CDUMP} -l select=1:ncpus=1:mem=$MEM -- ${this_dir}/get_v14.data.sh ${CDUMP}) DEPEND="-W depend=afterok:$DATAH" if [ "$CDUMP" = "gdas" ] ; then DATA1=$(qsub -V -o log.data.enkf -e log.data.enkf -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_enkf -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v14.data.sh enkf) + -N get_enkf -l select=1:ncpus=1:mem=$MEM -- ${this_dir}/get_v14.data.sh enkf) DEPEND="-W depend=afterok:$DATAH:$DATA1" fi ;; v15) if [ "$CDUMP" = "gfs" ] ; then DATAH=$(qsub -V -o log.data.${CDUMP} -e log.data.${CDUMP} -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_${CDUMP} -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh ${CDUMP}) + -N get_${CDUMP} -l select=1:ncpus=1:mem=$MEM -- ${this_dir}/get_v15.data.sh ${CDUMP}) DEPEND="-W depend=afterok:$DATAH" else DATAH=$(qsub -V -o log.data.${CDUMP} -e log.data.${CDUMP} -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_${CDUMP} -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh ${CDUMP}) + -N get_${CDUMP} -l select=1:ncpus=1:mem=$MEM -- ${this_dir}/get_v15.data.sh ${CDUMP}) DATA1=$(qsub -V -o log.data.grp1 -e log.data.grp1 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_grp1 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh grp1) + -N get_grp1 -l select=1:ncpus=1:mem=$MEM -- ${this_dir}/get_v15.data.sh grp1) DATA2=$(qsub -V -o log.data.grp2 -e log.data.grp2 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_grp2 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh grp2) + -N get_grp2 -l select=1:ncpus=1:mem=$MEM -- ${this_dir}/get_v15.data.sh grp2) DATA3=$(qsub -V -o log.data.grp3 -e log.data.grp3 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_grp3 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh grp3) + -N get_grp3 -l select=1:ncpus=1:mem=$MEM -- ${this_dir}/get_v15.data.sh grp3) DATA4=$(qsub -V -o log.data.grp4 -e log.data.grp4 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_grp4 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh grp4) + -N get_grp4 -l select=1:ncpus=1:mem=$MEM -- ${this_dir}/get_v15.data.sh grp4) DATA5=$(qsub -V -o log.data.grp5 -e log.data.grp5 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_grp5 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh grp5) + -N get_grp5 -l select=1:ncpus=1:mem=$MEM -- ${this_dir}/get_v15.data.sh grp5) DATA6=$(qsub -V -o log.data.grp6 -e log.data.grp6 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_grp6 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh grp6) + -N get_grp6 -l select=1:ncpus=1:mem=$MEM -- ${this_dir}/get_v15.data.sh grp6) DATA7=$(qsub -V -o log.data.grp7 -e log.data.grp7 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_grp7 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh grp7) + -N get_grp7 -l select=1:ncpus=1:mem=$MEM -- ${this_dir}/get_v15.data.sh grp7) DATA8=$(qsub -V -o log.data.grp8 -e log.data.grp8 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_grp8 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh grp8) + -N get_grp8 -l select=1:ncpus=1:mem=$MEM -- ${this_dir}/get_v15.data.sh grp8) DEPEND="-W depend=afterok:$DATAH:$DATA1:$DATA2:$DATA3:$DATA4:$DATA5:$DATA6:$DATA7:$DATA8" fi ;; v16retro) DATAH=$(qsub -V -o log.data.v16retro -e log.data.v16retro -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_v16retro -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16retro.data.sh ${CDUMP}) + -N get_v16retro -l select=1:ncpus=1:mem=$MEM -- ${this_dir}/get_v16retro.data.sh ${CDUMP}) DEPEND="-W depend=afterok:$DATAH" ;; v16) DATAH=$(qsub -V -o log.data.${CDUMP} -e log.data.${CDUMP} -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_${CDUMP} -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh ${CDUMP}) + -N get_${CDUMP} -l select=1:ncpus=1:mem=$MEM -- ${this_dir}/get_v16.data.sh ${CDUMP}) DEPEND="-W depend=afterok:$DATAH" if [ "$CDUMP" = "gdas" ] ; then DATA1=$(qsub -V -o log.data.grp1 -e log.data.grp1 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_grp1 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh grp1) + -N get_grp1 -l select=1:ncpus=1:mem=$MEM -- ${this_dir}/get_v16.data.sh grp1) DATA2=$(qsub -V -o log.data.grp2 -e log.data.grp2 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_grp2 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh grp2) + -N get_grp2 -l select=1:ncpus=1:mem=$MEM -- ${this_dir}/get_v16.data.sh grp2) DATA3=$(qsub -V -o log.data.grp3 -e log.data.grp3 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_grp3 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh grp3) + -N get_grp3 -l select=1:ncpus=1:mem=$MEM -- ${this_dir}/get_v16.data.sh grp3) DATA4=$(qsub -V -o log.data.grp4 -e log.data.grp4 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_grp4 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh grp4) + -N get_grp4 -l select=1:ncpus=1:mem=$MEM -- ${this_dir}/get_v16.data.sh grp4) DATA5=$(qsub -V -o log.data.grp5 -e log.data.grp5 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_grp5 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh grp5) + -N get_grp5 -l select=1:ncpus=1:mem=$MEM -- ${this_dir}/get_v16.data.sh grp5) DATA6=$(qsub -V -o log.data.grp6 -e log.data.grp6 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_grp5 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh grp6) + -N get_grp5 -l select=1:ncpus=1:mem=$MEM -- ${this_dir}/get_v16.data.sh grp6) DATA7=$(qsub -V -o log.data.grp7 -e log.data.grp7 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_grp7 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh grp7) + -N get_grp7 -l select=1:ncpus=1:mem=$MEM -- ${this_dir}/get_v16.data.sh grp7) DATA8=$(qsub -V -o log.data.grp8 -e log.data.grp8 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \ - -N get_grp8 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh grp8) + -N get_grp8 -l select=1:ncpus=1:mem=$MEM -- ${this_dir}/get_v16.data.sh grp8) DEPEND="-W depend=afterok:$DATAH:$DATA1:$DATA2:$DATA3:$DATA4:$DATA5:$DATA6:$DATA7:$DATA8" fi ;; @@ -132,7 +132,7 @@ if [ $RUN_CHGRES == yes ]; then TASKS_PER_NODE=48 WALLT="0:20:00" fi - (( NCPUS = TASKS_PER_NODE * 2 )) + NCPUS=${TASKS_PER_NODE} (( TASKS = NODES * TASKS_PER_NODE )) export APRUN="mpiexec -n $TASKS -ppn $TASKS_PER_NODE --cpu-bind core" case $gfs_ver in @@ -140,7 +140,7 @@ if [ $RUN_CHGRES == yes ]; then export OMP_NUM_THREADS=4 export OMP_STACKSIZE=1024M export OMP_PLACES=cores - export APRUN="$APRUN --depth 2" + export APRUN="$APRUN --depth ${OMP_NUM_THREADS}" (( NCPUS = NCPUS * OMP_NUM_THREADS )) qsub -V -l select=${NODES}:ncpus=${NCPUS}:ompthreads=${OMP_NUM_THREADS}:mem=${MEM} -l walltime=$WALLT -A $PROJECT_CODE -q $QUEUE \ -N chgres_${CDUMP} -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} -- ${this_dir}/run_pre-v14.chgres.sh ${CDUMP} @@ -179,7 +179,7 @@ if [ $RUN_CHGRES == yes ]; then MEM=75GB NODES=1 TASKS_PER_NODE=12 - (( NCPUS = TASKS_PER_NODE * 2 )) + NCPUS=${TASKS_PER_NODE} (( TASKS = NODES * TASKS_PER_NODE)) export APRUN="mpiexec -n $TASKS -ppn $TASKS_PER_NODE --cpu-bind core" @@ -195,7 +195,7 @@ if [ $RUN_CHGRES == yes ]; then export OMP_NUM_THREADS=2 export OMP_STACKSIZE=1024M export OMP_PLACES=cores - export APRUN="$APRUN --depth 2" + export APRUN="$APRUN --depth ${OMP_NUM_THREADS}" (( NCPUS = NCPUS * OMP_NUM_THREADS )) ;; esac From d0c77845422b1f0c6a9385f454f93b1d52499f08 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Mon, 27 Sep 2021 20:32:26 +0000 Subject: [PATCH 21/57] Update ./util/gdas_init/driver.wcoss2.sh to process all 80 enkf members. Fixes #559 --- util/gdas_init/driver.wcoss2.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/gdas_init/driver.wcoss2.sh b/util/gdas_init/driver.wcoss2.sh index 899532daa..2dcf65d7f 100755 --- a/util/gdas_init/driver.wcoss2.sh +++ b/util/gdas_init/driver.wcoss2.sh @@ -201,7 +201,7 @@ if [ $RUN_CHGRES == yes ]; then esac MEMBER=1 - while [ $MEMBER -le 04 ]; do + while [ $MEMBER -le 80 ]; do if [ $MEMBER -lt 10 ]; then MEMBER_CH="00${MEMBER}" else From 52a51922a3734c7b0a46aff1d5351ffc051447cf Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 30 Sep 2021 14:05:50 +0000 Subject: [PATCH 22/57] Update ice blend test to use v1.2.3 of grib_util. v1.2.2 gave wrong results when replacing the bitmap with the 1.57 flag value. Fixes #559. --- reg_tests/ice_blend/driver.wcoss2.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reg_tests/ice_blend/driver.wcoss2.sh b/reg_tests/ice_blend/driver.wcoss2.sh index c58062ae5..6d03a0b91 100755 --- a/reg_tests/ice_blend/driver.wcoss2.sh +++ b/reg_tests/ice_blend/driver.wcoss2.sh @@ -31,7 +31,7 @@ cd $PBS_O_WORKDIR source ../../sorc/machine-setup.sh > /dev/null 2>&1 module use ../../modulefiles module load build.$target.intel -module load grib_util/1.2.2 +module load grib_util/1.2.3 module list set -x From d71d386fff5ec19c605db5fe8b660efbb22db9c9 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Fri, 1 Oct 2021 19:07:35 +0000 Subject: [PATCH 23/57] Update build module to point to latest esmf v8. Fixes #559 --- modulefiles/build.wcoss2_cray.intel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/build.wcoss2_cray.intel b/modulefiles/build.wcoss2_cray.intel index a40f1ef1b..a5c9a44b9 100644 --- a/modulefiles/build.wcoss2_cray.intel +++ b/modulefiles/build.wcoss2_cray.intel @@ -17,7 +17,7 @@ module load libpng/1.6.37 module load hdf5/1.10.6 module load netcdf/4.7.4 -module load esmf/8_1_0_beta_snapshot_27 +module load esmf/8.1.1 module load bacio/2.4.1 module load sfcio/1.4.1 From 4468002a4681fb02f08ebda2683fe595ec51e7b9 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Fri, 1 Oct 2021 19:10:11 +0000 Subject: [PATCH 24/57] Update consistency test driver scripts to point to my local copy of 'nccmp'. Hope this is temporary. Fixes #559. --- reg_tests/chgres_cube/driver.wcoss2.sh | 5 +++-- reg_tests/global_cycle/driver.wcoss2.sh | 2 ++ reg_tests/grid_gen/driver.wcoss2.sh | 2 ++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/reg_tests/chgres_cube/driver.wcoss2.sh b/reg_tests/chgres_cube/driver.wcoss2.sh index e7eed0333..c8cf36f99 100755 --- a/reg_tests/chgres_cube/driver.wcoss2.sh +++ b/reg_tests/chgres_cube/driver.wcoss2.sh @@ -53,7 +53,8 @@ rm -f $LOG_FILE* $SUM_FILE export OMP_STACKSIZE=1024M -export NCCMP=${NCCMP:-nccmp} +export NCCMP=/lfs/h2/emc/eib/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp +#export NCCMP=${NCCMP:-nccmp} rm -fr $OUTDIR this_dir=$PWD @@ -183,7 +184,7 @@ TEST13=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltim LOG_FILE=consistency.log14 export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" TEST14=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N 13km.na.gfs.ncei.grib2.conus -l select=1:ncpus=6:ompthreads=1:mem=15GB $PWD/13km.na.gfs.ncei.grib2.sh) + -N 13km.na.gfs.ncei.grib2.conus -l select=1:ncpus=6:ompthreads=1:mem=25GB $PWD/13km.na.gfs.ncei.grib2.sh) #----------------------------------------------------------------------------- # Initialize C96 WAM IC using FV3 gaussian netcdf files. diff --git a/reg_tests/global_cycle/driver.wcoss2.sh b/reg_tests/global_cycle/driver.wcoss2.sh index 6dba4e96f..a2a58a617 100755 --- a/reg_tests/global_cycle/driver.wcoss2.sh +++ b/reg_tests/global_cycle/driver.wcoss2.sh @@ -49,6 +49,8 @@ export NWPROD=$PWD/../.. reg_dir=$PWD +export NCCMP=/lfs/h2/emc/eib/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp + LOG_FILE=consistency.log rm -f ${LOG_FILE}* diff --git a/reg_tests/grid_gen/driver.wcoss2.sh b/reg_tests/grid_gen/driver.wcoss2.sh index 17f817f33..e932b3ac8 100755 --- a/reg_tests/grid_gen/driver.wcoss2.sh +++ b/reg_tests/grid_gen/driver.wcoss2.sh @@ -38,6 +38,8 @@ PROJECT_CODE="${PROJECT_CODE:-GFS-DEV}" # Should not have to change anything below here. #----------------------------------------------------------------------------- +export NCCMP=/lfs/h2/emc/eib/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp + LOG_FILE=consistency.log rm -f ${LOG_FILE} SUM_FILE=summary.log From 8256d77cbc69524c8abf93400b62cbf9f5138b6f Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Fri, 15 Oct 2021 19:04:14 +0000 Subject: [PATCH 25/57] Update fix/link_fixdirs.sh to point to the correct 'fix' directories. Fixes #559. --- fix/link_fixdirs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fix/link_fixdirs.sh b/fix/link_fixdirs.sh index d848dc6d0..32d2b8efa 100755 --- a/fix/link_fixdirs.sh +++ b/fix/link_fixdirs.sh @@ -46,7 +46,7 @@ elif [ $machine = "jet" ]; then elif [ $machine = "orion" ]; then FIX_DIR="/work/noaa/global/glopara/fix" elif [ $machine = "wcoss2" ]; then - FIX_DIR="/lfs/h2/emc/eib/noscrub/Kate.Friedman/glopara/fix_nco_gfsv16" + FIX_DIR="/lfs/h2/emc/global/noscrub/Kate.Friedman/glopara/FIX/fix" elif [ $machine = "s4" ]; then FIX_DIR="/data/prod/glopara/fix" fi From 426ba04bbb29d9d7ee8cc7ec6afe17ffe3122322 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Fri, 15 Oct 2021 20:59:56 +0000 Subject: [PATCH 26/57] Point to new location for data and utilities on wcoss2. Fixes #559. --- reg_tests/chgres_cube/driver.wcoss2.sh | 4 ++-- reg_tests/global_cycle/driver.wcoss2.sh | 4 ++-- reg_tests/grid_gen/driver.wcoss2.sh | 4 ++-- reg_tests/ice_blend/driver.wcoss2.sh | 2 +- reg_tests/snow2mdl/driver.wcoss2.sh | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/reg_tests/chgres_cube/driver.wcoss2.sh b/reg_tests/chgres_cube/driver.wcoss2.sh index c8cf36f99..ce817cd9e 100755 --- a/reg_tests/chgres_cube/driver.wcoss2.sh +++ b/reg_tests/chgres_cube/driver.wcoss2.sh @@ -45,7 +45,7 @@ QUEUE="${QUEUE:-dev}" export HOMEufs=$PWD/../.. -export HOMEreg=/lfs/h2/emc/eib/noscrub/George.Gayno/ufs_utils.git/reg_tests/chgres_cube +export HOMEreg=/lfs/h2/emc/global/noscrub/George.Gayno/ufs_utils.git/reg_tests/chgres_cube LOG_FILE=consistency.log SUM_FILE=summary.log @@ -53,7 +53,7 @@ rm -f $LOG_FILE* $SUM_FILE export OMP_STACKSIZE=1024M -export NCCMP=/lfs/h2/emc/eib/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp +export NCCMP=/lfs/h2/emc/global/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp #export NCCMP=${NCCMP:-nccmp} rm -fr $OUTDIR diff --git a/reg_tests/global_cycle/driver.wcoss2.sh b/reg_tests/global_cycle/driver.wcoss2.sh index a2a58a617..419daab05 100755 --- a/reg_tests/global_cycle/driver.wcoss2.sh +++ b/reg_tests/global_cycle/driver.wcoss2.sh @@ -38,7 +38,7 @@ QUEUE="${QUEUE:-dev}" DATA_DIR="${WORK_DIR}/reg-tests/global-cycle" -export HOMEreg=/lfs/h2/emc/eib/noscrub/George.Gayno/ufs_utils.git/reg_tests/global_cycle +export HOMEreg=/lfs/h2/emc/global/noscrub/George.Gayno/ufs_utils.git/reg_tests/global_cycle export OMP_NUM_THREADS_CY=2 export OMP_PLACES=cores @@ -49,7 +49,7 @@ export NWPROD=$PWD/../.. reg_dir=$PWD -export NCCMP=/lfs/h2/emc/eib/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp +export NCCMP=/lfs/h2/emc/global/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp LOG_FILE=consistency.log rm -f ${LOG_FILE}* diff --git a/reg_tests/grid_gen/driver.wcoss2.sh b/reg_tests/grid_gen/driver.wcoss2.sh index e932b3ac8..2f2bdeb2d 100755 --- a/reg_tests/grid_gen/driver.wcoss2.sh +++ b/reg_tests/grid_gen/driver.wcoss2.sh @@ -38,7 +38,7 @@ PROJECT_CODE="${PROJECT_CODE:-GFS-DEV}" # Should not have to change anything below here. #----------------------------------------------------------------------------- -export NCCMP=/lfs/h2/emc/eib/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp +export NCCMP=/lfs/h2/emc/global/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp LOG_FILE=consistency.log rm -f ${LOG_FILE} @@ -50,7 +50,7 @@ export OMP_STACKSIZE=2048m export OMP_NUM_THREADS=30 # orog code uses threads export OMP_PLACES=cores export machine=WCOSS2 -export HOMEreg=/lfs/h2/emc/eib/noscrub/George.Gayno/ufs_utils.git/reg_tests/grid_gen/baseline_data +export HOMEreg=/lfs/h2/emc/global/noscrub/George.Gayno/ufs_utils.git/reg_tests/grid_gen/baseline_data this_dir=$PWD ulimit -a diff --git a/reg_tests/ice_blend/driver.wcoss2.sh b/reg_tests/ice_blend/driver.wcoss2.sh index 6d03a0b91..fb0eb054c 100755 --- a/reg_tests/ice_blend/driver.wcoss2.sh +++ b/reg_tests/ice_blend/driver.wcoss2.sh @@ -43,7 +43,7 @@ export DATA="${DATA}/reg-tests/ice-blend" # Should not have to change anything below. #----------------------------------------------------------------------------- -export HOMEreg=/lfs/h2/emc/eib/noscrub/George.Gayno/ufs_utils.git/reg_tests/ice_blend +export HOMEreg=/lfs/h2/emc/global/noscrub/George.Gayno/ufs_utils.git/reg_tests/ice_blend export HOMEgfs=$PBS_O_WORKDIR/../.. rm -fr $DATA diff --git a/reg_tests/snow2mdl/driver.wcoss2.sh b/reg_tests/snow2mdl/driver.wcoss2.sh index cc792ee34..e0596bf4b 100755 --- a/reg_tests/snow2mdl/driver.wcoss2.sh +++ b/reg_tests/snow2mdl/driver.wcoss2.sh @@ -43,7 +43,7 @@ export DATA="${DATA}/reg-tests/snow2mdl" # Should not have to change anything below. #----------------------------------------------------------------------------- -export HOMEreg=/lfs/h2/emc/eib/noscrub/George.Gayno/ufs_utils.git/reg_tests/snow2mdl +export HOMEreg=/lfs/h2/emc/global/noscrub/George.Gayno/ufs_utils.git/reg_tests/snow2mdl export HOMEgfs=$PWD/../.. rm -fr $DATA From e6c3042c3b6025a01f6dc0f9c038596002aa0a2e Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 21 Oct 2021 17:20:27 +0000 Subject: [PATCH 27/57] Remove hard-wired library versions per NCO directive for WCOSS2. Fixes #559. --- build_all.sh | 8 +++- driver_scripts/driver_grid.wcoss2.sh | 3 +- modulefiles/build.wcoss2_cray.intel | 50 ++++++++++++------------- reg_tests/chgres_cube/driver.wcoss2.sh | 3 +- reg_tests/global_cycle/driver.wcoss2.sh | 3 +- reg_tests/grid_gen/driver.wcoss2.sh | 3 +- reg_tests/ice_blend/driver.wcoss2.sh | 3 +- reg_tests/snow2mdl/driver.wcoss2.sh | 3 +- util/gdas_init/driver.wcoss2.sh | 3 +- util/sfc_climo_gen/run.wcoss2.sh | 3 +- util/vcoord_gen/run.wcoss2.sh | 3 +- 11 files changed, 40 insertions(+), 45 deletions(-) diff --git a/build_all.sh b/build_all.sh index fe457c9ca..310aeca24 100755 --- a/build_all.sh +++ b/build_all.sh @@ -20,8 +20,12 @@ if [[ "$target" == "linux.*" || "$target" == "macosx.*" ]]; then else set +x source ./sorc/machine-setup.sh - module use ./modulefiles - module load build.$target.$compiler > /dev/null + if [[ "$target" == "wcoss2_cray" ]];then + source ./modulefiles/build.$target.$compiler + else + module use ./modulefiles + module load build.$target.$compiler > /dev/null + fi module list set -x fi diff --git a/driver_scripts/driver_grid.wcoss2.sh b/driver_scripts/driver_grid.wcoss2.sh index 90eac38b3..caea36591 100755 --- a/driver_scripts/driver_grid.wcoss2.sh +++ b/driver_scripts/driver_grid.wcoss2.sh @@ -61,8 +61,7 @@ cd $PBS_O_WORKDIR source ../sorc/machine-setup.sh > /dev/null 2>&1 -module use ../modulefiles -module load build.$target.intel +source ../modulefiles/build.$target.intel module list #----------------------------------------------------------------------- diff --git a/modulefiles/build.wcoss2_cray.intel b/modulefiles/build.wcoss2_cray.intel index a5c9a44b9..038181440 100644 --- a/modulefiles/build.wcoss2_cray.intel +++ b/modulefiles/build.wcoss2_cray.intel @@ -1,33 +1,33 @@ -#%Module##################################################### +#!/bin/bash + ## Build and run module for WCOSS2-Cray -############################################################# -module load envvar/1.0 -module load cmake/3.20.2 +module load envvar/${envvar_ver:-1.0} +module load cmake/${cmake_ver:-3.20.2} -module load PrgEnv-intel/8.1.0 -module load craype/2.7.8 -module load intel/19.1.3.304 -module load cray-mpich/8.1.7 +module load PrgEnv-intel/${PrgEnv_intel_ver:-8.1.0} +module load craype/${craype_ver:-2.7.8} +module load intel/${intel_ver:-19.1.3.304} +module load cray-mpich/${cray_mpich_ver:-8.1.7} -module load libjpeg/9c -module load jasper/2.0.25 -module load zlib/1.2.11 -module load libpng/1.6.37 +module load libjpeg/${libjpeg_ver:-9c} +module load jasper/${jasper_ver:-2.0.25} +module load zlib/${zlib_ver:-1.2.11} +module load libpng/${libpng_ver:-1.6.37} -module load hdf5/1.10.6 -module load netcdf/4.7.4 -module load esmf/8.1.1 +module load hdf5/${hdf5_ver:-1.10.6} +module load netcdf/${netcdf_ver:-4.7.4} +module load esmf/${esmf_ver:-8.1.1} -module load bacio/2.4.1 -module load sfcio/1.4.1 -module load w3nco/2.4.1 -module load nemsio/2.5.2 -module load sigio/2.3.2 -module load sp/2.3.3 -module load ip/3.3.3 -module load g2/3.4.1 -module load wgrib2/2.0.8 +module load bacio/${bacio_ver:-2.4.1} +module load sfcio/${sfcio_ver:-1.4.1} +module load w3nco/${w3nco_ver:-2.4.1} +module load nemsio/${nemsio_ver:-2.5.2} +module load sigio/${sigio_ver:-2.3.2} +module load sp/${sp_ver:-2.3.3} +module load ip/${ip_ver:-3.3.3} +module load g2/${g2_ver:-3.4.1} +module load wgrib2/${wgrib2_ver:-2.0.8} # for mpiexec command -module load cray-pals/1.0.12 +module load cray-pals/${cray_pals_ver:-1.0.12} diff --git a/reg_tests/chgres_cube/driver.wcoss2.sh b/reg_tests/chgres_cube/driver.wcoss2.sh index ce817cd9e..08595caef 100755 --- a/reg_tests/chgres_cube/driver.wcoss2.sh +++ b/reg_tests/chgres_cube/driver.wcoss2.sh @@ -27,8 +27,7 @@ set -x compiler=${compiler:-"intel"} source ../../sorc/machine-setup.sh > /dev/null 2>&1 -module use ../../modulefiles -module load build.$target.$compiler +source ../../modulefiles/build.$target.$compiler module list export OUTDIR="${WORK_DIR:-/lfs/h2/emc/stmp/$LOGNAME}" diff --git a/reg_tests/global_cycle/driver.wcoss2.sh b/reg_tests/global_cycle/driver.wcoss2.sh index 419daab05..16ab8601f 100755 --- a/reg_tests/global_cycle/driver.wcoss2.sh +++ b/reg_tests/global_cycle/driver.wcoss2.sh @@ -23,8 +23,7 @@ set -x compiler=${compiler:-"intel"} source ../../sorc/machine-setup.sh > /dev/null 2>&1 -module use ../../modulefiles -module load build.$target.$compiler +source ../../modulefiles/build.$target.$compiler module list WORK_DIR="${WORK_DIR:-/lfs/h2/emc/stmp/$LOGNAME}" diff --git a/reg_tests/grid_gen/driver.wcoss2.sh b/reg_tests/grid_gen/driver.wcoss2.sh index 2f2bdeb2d..55d4d7486 100755 --- a/reg_tests/grid_gen/driver.wcoss2.sh +++ b/reg_tests/grid_gen/driver.wcoss2.sh @@ -23,8 +23,7 @@ compiler=${compiler:-"intel"} source ../../sorc/machine-setup.sh > /dev/null 2>&1 -module use ../../modulefiles -module load build.$target.$compiler +source ../../modulefiles/build.$target.$compiler module list set -x diff --git a/reg_tests/ice_blend/driver.wcoss2.sh b/reg_tests/ice_blend/driver.wcoss2.sh index fb0eb054c..78a615817 100755 --- a/reg_tests/ice_blend/driver.wcoss2.sh +++ b/reg_tests/ice_blend/driver.wcoss2.sh @@ -29,8 +29,7 @@ cd $PBS_O_WORKDIR source ../../sorc/machine-setup.sh > /dev/null 2>&1 -module use ../../modulefiles -module load build.$target.intel +source ../../modulefiles/build.$target.intel module load grib_util/1.2.3 module list diff --git a/reg_tests/snow2mdl/driver.wcoss2.sh b/reg_tests/snow2mdl/driver.wcoss2.sh index e0596bf4b..f196c25d9 100755 --- a/reg_tests/snow2mdl/driver.wcoss2.sh +++ b/reg_tests/snow2mdl/driver.wcoss2.sh @@ -29,8 +29,7 @@ cd $PBS_O_WORKDIR source ../../sorc/machine-setup.sh > /dev/null 2>&1 -module use ../../modulefiles -module load build.$target.intel +source ../../modulefiles/build.$target.intel module load grib_util/1.2.2 module list diff --git a/util/gdas_init/driver.wcoss2.sh b/util/gdas_init/driver.wcoss2.sh index 2dcf65d7f..0a41f7fa2 100755 --- a/util/gdas_init/driver.wcoss2.sh +++ b/util/gdas_init/driver.wcoss2.sh @@ -10,8 +10,7 @@ set -x compiler=${compiler:-"intel"} source ../../sorc/machine-setup.sh > /dev/null 2>&1 -module use ../../modulefiles -module load build.$target.$compiler +source ../../modulefiles/build.$target.$compiler module list # Needed for NDATE utility diff --git a/util/sfc_climo_gen/run.wcoss2.sh b/util/sfc_climo_gen/run.wcoss2.sh index 63e9c692f..1ba479c97 100755 --- a/util/sfc_climo_gen/run.wcoss2.sh +++ b/util/sfc_climo_gen/run.wcoss2.sh @@ -23,8 +23,7 @@ export APRUN_SFC="mpiexec -n 24 -ppn 24 -cpu-bind core" export BASE_DIR=$PBS_O_WORKDIR/../.. source ${BASE_DIR}/sorc/machine-setup.sh > /dev/null 2>&1 -module use ${BASE_DIR}/modulefiles -module load build.$target.intel +source ${BASE_DIR}/modulefiles/build.$target.intel module list #------------------------------------- diff --git a/util/vcoord_gen/run.wcoss2.sh b/util/vcoord_gen/run.wcoss2.sh index 7a53c8533..b0eedd729 100755 --- a/util/vcoord_gen/run.wcoss2.sh +++ b/util/vcoord_gen/run.wcoss2.sh @@ -25,8 +25,7 @@ cd $PBS_O_WORKDIR set -x source ../../sorc/machine-setup.sh > /dev/null 2>&1 -module use ../../modulefiles -module load build.$target.intel +source ../../modulefiles/build.$target.intel module list outfile="./global_hyblev.txt" From 8902c900ab188e7ab92354a7971a0653a4f65cbe Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 2 Dec 2021 21:10:36 +0000 Subject: [PATCH 28/57] Change references 'wcoss2_cray' to 'wcoss2'. Fixes #559. --- build_all.sh | 2 +- modulefiles/{build.wcoss2_cray.intel => build.wcoss2.intel} | 0 sorc/machine-setup.sh | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename modulefiles/{build.wcoss2_cray.intel => build.wcoss2.intel} (100%) diff --git a/build_all.sh b/build_all.sh index 310aeca24..7070bcf7e 100755 --- a/build_all.sh +++ b/build_all.sh @@ -20,7 +20,7 @@ if [[ "$target" == "linux.*" || "$target" == "macosx.*" ]]; then else set +x source ./sorc/machine-setup.sh - if [[ "$target" == "wcoss2_cray" ]];then + if [[ "$target" == "wcoss2" ]];then source ./modulefiles/build.$target.$compiler else module use ./modulefiles diff --git a/modulefiles/build.wcoss2_cray.intel b/modulefiles/build.wcoss2.intel similarity index 100% rename from modulefiles/build.wcoss2_cray.intel rename to modulefiles/build.wcoss2.intel diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh index d8294f2e5..3bd60c6c7 100644 --- a/sorc/machine-setup.sh +++ b/sorc/machine-setup.sh @@ -28,7 +28,7 @@ if [[ -d /lfs3 ]] ; then target=jet module purge elif [[ -d /lfs/h1 ]] ; then - target=wcoss2_cray + target=wcoss2 module purge elif [[ -d /scratch1 ]] ; then # We are on NOAA Hera From c471fd714973fe5e3d8f165334d60c80dfca367e Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 2 Dec 2021 21:55:06 +0000 Subject: [PATCH 29/57] Update wcoss2 build module to be lua compliant. Fixes #559. --- modulefiles/{build.wcoss2.intel => build.wcoss2.intel.lua} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename modulefiles/{build.wcoss2.intel => build.wcoss2.intel.lua} (100%) diff --git a/modulefiles/build.wcoss2.intel b/modulefiles/build.wcoss2.intel.lua similarity index 100% rename from modulefiles/build.wcoss2.intel rename to modulefiles/build.wcoss2.intel.lua From eb934cd7eb92b87258762a175da261a108d5c4d6 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Fri, 3 Dec 2021 14:58:25 +0000 Subject: [PATCH 30/57] More updates to wcoss2 build process for lua compliance. Fixes #559. --- build_all.sh | 8 +-- modulefiles/build.wcoss2.intel.lua | 105 ++++++++++++++++++++--------- 2 files changed, 74 insertions(+), 39 deletions(-) diff --git a/build_all.sh b/build_all.sh index 7070bcf7e..fe457c9ca 100755 --- a/build_all.sh +++ b/build_all.sh @@ -20,12 +20,8 @@ if [[ "$target" == "linux.*" || "$target" == "macosx.*" ]]; then else set +x source ./sorc/machine-setup.sh - if [[ "$target" == "wcoss2" ]];then - source ./modulefiles/build.$target.$compiler - else - module use ./modulefiles - module load build.$target.$compiler > /dev/null - fi + module use ./modulefiles + module load build.$target.$compiler > /dev/null module list set -x fi diff --git a/modulefiles/build.wcoss2.intel.lua b/modulefiles/build.wcoss2.intel.lua index 038181440..8c5f0f0f5 100644 --- a/modulefiles/build.wcoss2.intel.lua +++ b/modulefiles/build.wcoss2.intel.lua @@ -1,33 +1,72 @@ -#!/bin/bash - -## Build and run module for WCOSS2-Cray - -module load envvar/${envvar_ver:-1.0} -module load cmake/${cmake_ver:-3.20.2} - -module load PrgEnv-intel/${PrgEnv_intel_ver:-8.1.0} -module load craype/${craype_ver:-2.7.8} -module load intel/${intel_ver:-19.1.3.304} -module load cray-mpich/${cray_mpich_ver:-8.1.7} - -module load libjpeg/${libjpeg_ver:-9c} -module load jasper/${jasper_ver:-2.0.25} -module load zlib/${zlib_ver:-1.2.11} -module load libpng/${libpng_ver:-1.6.37} - -module load hdf5/${hdf5_ver:-1.10.6} -module load netcdf/${netcdf_ver:-4.7.4} -module load esmf/${esmf_ver:-8.1.1} - -module load bacio/${bacio_ver:-2.4.1} -module load sfcio/${sfcio_ver:-1.4.1} -module load w3nco/${w3nco_ver:-2.4.1} -module load nemsio/${nemsio_ver:-2.5.2} -module load sigio/${sigio_ver:-2.3.2} -module load sp/${sp_ver:-2.3.3} -module load ip/${ip_ver:-3.3.3} -module load g2/${g2_ver:-3.4.1} -module load wgrib2/${wgrib2_ver:-2.0.8} - -# for mpiexec command -module load cray-pals/${cray_pals_ver:-1.0.12} +help([[ +Load environment to compile UFS_UTILS on WCOSS2 +]]) + +envvar_ver=os.getenv("envvar_ver") or "1.0" +load(pathJoin("envvar", envvar_ver)) + +cmake_ver=os.getenv("cmake_ver") or "3.20.2" +load(pathJoin("cmake", cmake_ver)) + +PrgEnv_intel_ver=os.getenv("PrgEnv_intel_ver") or "8.1.0" +load(pathJoin("PrgEnv-intel", PrgEnv_intel_ver)) + +craype_ver=os.getenv("craype_ver") or "2.7.8" +load(pathJoin("craype", craype_ver)) + +intel_ver=os.getenv("intel_ver") or "19.1.3.304" +load(pathJoin("intel", intel_ver)) + +cray_mpich_ver=os.getenv("cray_mpich_ver") or "8.1.7" +load(pathJoin("cray-mpich", cray_mpich_ver)) + +libjpeg_ver=os.getenv("libjpeg_ver") or "9c" +load(pathJoin("libjpeg", libjpeg_ver)) + +zlib_ver=os.getenv("zlib_ver") or "1.2.11" +load(pathJoin("zlib", zlib_ver)) + +libpng_ver=os.getenv("libpng_ver") or "1.6.37" +load(pathJoin("libpng", libpng_ver)) + +hdf5_ver=os.getenv("hdf5_ver") or "1.10.6" +load(pathJoin("hdf5", hdf5_ver)) + +netcdf_ver=os.getenv("netcdf_ver") or "4.7.4" +load(pathJoin("netcdf", netcdf_ver)) + +esmf_ver=os.getenv("esmf_ver") or "8.1.1" +load(pathJoin("esmf", esmf_ver)) + +bacio_ver=os.getenv("bacio_ver") or "2.4.1" +load(pathJoin("bacio", bacio_ver)) + +sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" +load(pathJoin("sfcio", sfcio_ver)) + +w3nco_ver=os.getenv("w3nco_ver") or "2.4.1" +load(pathJoin("w3nco", w3nco_ver)) + +nemsio_ver=os.getenv("nemsio_ver") or "2.5.2" +load(pathJoin("nemsio", nemsio_ver)) + +sigio_ver=os.getenv("sigio_ver") or "2.3.2" +load(pathJoin("sigio", sigio_ver)) + +sp_ver=os.getenv("sp_ver") or "2.3.3" +load(pathJoin("sp", sp_ver)) + +ip_ver=os.getenv("ip_ver") or "3.3.3" +load(pathJoin("ip", ip_ver)) + +g2_ver=os.getenv("g2_ver") or "3.4.1" +load(pathJoin("g2", g2_ver)) + +wgrib2_ver=os.getenv("wgrib2_ver") or "2.0.8" +load(pathJoin("wgrib2", wgrib2_ver)) + +-- for mpiexec command +cray_pals_ver=os.getenv("cray_pals_ver") or "1.0.12" +load(pathJoin("cray-pals", cray_pals_ver)) + +whatis("Description: UFS_UTILS build environment") From 84df02afc15a112f2385fe3d465c28198025affc Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Fri, 3 Dec 2021 15:52:38 +0000 Subject: [PATCH 31/57] Update utility scripts for new lua based build module. Fixes #559. --- util/gdas_init/config | 4 ++-- util/gdas_init/driver.wcoss2.sh | 3 ++- util/sfc_climo_gen/run.wcoss2.sh | 3 ++- util/vcoord_gen/run.wcoss2.sh | 5 +++-- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/util/gdas_init/config b/util/gdas_init/config index ddfc7c025..d31108fac 100644 --- a/util/gdas_init/config +++ b/util/gdas_init/config @@ -47,7 +47,7 @@ # #----------------------------------------------------------- -EXTRACT_DIR=/lfs4/HFIP/emcda/$USER/stmp/gdas.init/input +EXTRACT_DIR=/lfs/h2/emc/stmp/$USER/gdas.init/input EXTRACT_DATA=yes RUN_CHGRES=no @@ -68,7 +68,7 @@ CRES_ENKF=C96 UFS_DIR=$PWD/../.. -OUTDIR=/lfs4/HFIP/emcda/$USER/stmp/gdas.init/output +OUTDIR=/lfs/h2/emc/stmp/$USER/gdas.init/output #--------------------------------------------------------- # Dont touch anything below here. diff --git a/util/gdas_init/driver.wcoss2.sh b/util/gdas_init/driver.wcoss2.sh index 0a41f7fa2..2dcf65d7f 100755 --- a/util/gdas_init/driver.wcoss2.sh +++ b/util/gdas_init/driver.wcoss2.sh @@ -10,7 +10,8 @@ set -x compiler=${compiler:-"intel"} source ../../sorc/machine-setup.sh > /dev/null 2>&1 -source ../../modulefiles/build.$target.$compiler +module use ../../modulefiles +module load build.$target.$compiler module list # Needed for NDATE utility diff --git a/util/sfc_climo_gen/run.wcoss2.sh b/util/sfc_climo_gen/run.wcoss2.sh index 1ba479c97..63e9c692f 100755 --- a/util/sfc_climo_gen/run.wcoss2.sh +++ b/util/sfc_climo_gen/run.wcoss2.sh @@ -23,7 +23,8 @@ export APRUN_SFC="mpiexec -n 24 -ppn 24 -cpu-bind core" export BASE_DIR=$PBS_O_WORKDIR/../.. source ${BASE_DIR}/sorc/machine-setup.sh > /dev/null 2>&1 -source ${BASE_DIR}/modulefiles/build.$target.intel +module use ${BASE_DIR}/modulefiles +module load build.$target.intel module list #------------------------------------- diff --git a/util/vcoord_gen/run.wcoss2.sh b/util/vcoord_gen/run.wcoss2.sh index b0eedd729..0504db37c 100755 --- a/util/vcoord_gen/run.wcoss2.sh +++ b/util/vcoord_gen/run.wcoss2.sh @@ -6,7 +6,7 @@ #PBS -N vcoord #PBS -q debug #PBS -A GFS-DEV -#PBS -l select=1:ncpus=1:mem=1MB +#PBS -l select=1:ncpus=1:mem=20MB #------------------------------------------------------------------------------- # @@ -25,7 +25,8 @@ cd $PBS_O_WORKDIR set -x source ../../sorc/machine-setup.sh > /dev/null 2>&1 -source ../../modulefiles/build.$target.intel +module use ../../modulefiles +module load build.$target.intel module list outfile="./global_hyblev.txt" From 586372fab63e30fe05dc94da6bb35d917de71313 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Fri, 3 Dec 2021 16:30:20 +0000 Subject: [PATCH 32/57] Update regression test scripts for new lua-based build module. Fixes #559. --- reg_tests/chgres_cube/driver.wcoss2.sh | 3 ++- reg_tests/global_cycle/driver.wcoss2.sh | 3 ++- reg_tests/grid_gen/driver.wcoss2.sh | 3 ++- reg_tests/ice_blend/driver.wcoss2.sh | 3 ++- reg_tests/snow2mdl/driver.wcoss2.sh | 3 ++- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/reg_tests/chgres_cube/driver.wcoss2.sh b/reg_tests/chgres_cube/driver.wcoss2.sh index 08595caef..ce817cd9e 100755 --- a/reg_tests/chgres_cube/driver.wcoss2.sh +++ b/reg_tests/chgres_cube/driver.wcoss2.sh @@ -27,7 +27,8 @@ set -x compiler=${compiler:-"intel"} source ../../sorc/machine-setup.sh > /dev/null 2>&1 -source ../../modulefiles/build.$target.$compiler +module use ../../modulefiles +module load build.$target.$compiler module list export OUTDIR="${WORK_DIR:-/lfs/h2/emc/stmp/$LOGNAME}" diff --git a/reg_tests/global_cycle/driver.wcoss2.sh b/reg_tests/global_cycle/driver.wcoss2.sh index 16ab8601f..419daab05 100755 --- a/reg_tests/global_cycle/driver.wcoss2.sh +++ b/reg_tests/global_cycle/driver.wcoss2.sh @@ -23,7 +23,8 @@ set -x compiler=${compiler:-"intel"} source ../../sorc/machine-setup.sh > /dev/null 2>&1 -source ../../modulefiles/build.$target.$compiler +module use ../../modulefiles +module load build.$target.$compiler module list WORK_DIR="${WORK_DIR:-/lfs/h2/emc/stmp/$LOGNAME}" diff --git a/reg_tests/grid_gen/driver.wcoss2.sh b/reg_tests/grid_gen/driver.wcoss2.sh index 55d4d7486..2f2bdeb2d 100755 --- a/reg_tests/grid_gen/driver.wcoss2.sh +++ b/reg_tests/grid_gen/driver.wcoss2.sh @@ -23,7 +23,8 @@ compiler=${compiler:-"intel"} source ../../sorc/machine-setup.sh > /dev/null 2>&1 -source ../../modulefiles/build.$target.$compiler +module use ../../modulefiles +module load build.$target.$compiler module list set -x diff --git a/reg_tests/ice_blend/driver.wcoss2.sh b/reg_tests/ice_blend/driver.wcoss2.sh index 78a615817..fb0eb054c 100755 --- a/reg_tests/ice_blend/driver.wcoss2.sh +++ b/reg_tests/ice_blend/driver.wcoss2.sh @@ -29,7 +29,8 @@ cd $PBS_O_WORKDIR source ../../sorc/machine-setup.sh > /dev/null 2>&1 -source ../../modulefiles/build.$target.intel +module use ../../modulefiles +module load build.$target.intel module load grib_util/1.2.3 module list diff --git a/reg_tests/snow2mdl/driver.wcoss2.sh b/reg_tests/snow2mdl/driver.wcoss2.sh index f196c25d9..e0596bf4b 100755 --- a/reg_tests/snow2mdl/driver.wcoss2.sh +++ b/reg_tests/snow2mdl/driver.wcoss2.sh @@ -29,7 +29,8 @@ cd $PBS_O_WORKDIR source ../../sorc/machine-setup.sh > /dev/null 2>&1 -source ../../modulefiles/build.$target.intel +module use ../../modulefiles +module load build.$target.intel module load grib_util/1.2.2 module list From 54b1b447bee6cb963e83feecfd373ab7cce8d84d Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Fri, 3 Dec 2021 16:40:23 +0000 Subject: [PATCH 33/57] Update grid creation driver script for new lua-compliant build module. Fixes #559. --- driver_scripts/driver_grid.wcoss2.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/driver_scripts/driver_grid.wcoss2.sh b/driver_scripts/driver_grid.wcoss2.sh index caea36591..90eac38b3 100755 --- a/driver_scripts/driver_grid.wcoss2.sh +++ b/driver_scripts/driver_grid.wcoss2.sh @@ -61,7 +61,8 @@ cd $PBS_O_WORKDIR source ../sorc/machine-setup.sh > /dev/null 2>&1 -source ../modulefiles/build.$target.intel +module use ../modulefiles +module load build.$target.intel module list #----------------------------------------------------------------------- From 0ff75e9182dccfbbd3989231f6b90717ef660ba4 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Mon, 27 Dec 2021 20:16:07 +0000 Subject: [PATCH 34/57] Remove all references to jlogfile from scripts per NCO requirements. Fixes #559. --- scripts/exemcsfc_global_sfc_prep.sh | 47 ++++++----------------------- ush/emcsfc_ice_blend.sh | 24 ++++----------- ush/emcsfc_snow.sh | 21 +++---------- 3 files changed, 21 insertions(+), 71 deletions(-) diff --git a/scripts/exemcsfc_global_sfc_prep.sh b/scripts/exemcsfc_global_sfc_prep.sh index 7662402a9..9587f2a07 100755 --- a/scripts/exemcsfc_global_sfc_prep.sh +++ b/scripts/exemcsfc_global_sfc_prep.sh @@ -90,7 +90,7 @@ cd $DATA export SENDCOM=${SENDCOM:-"NO"} #----------------------------------------------------------------------- -# the "postmsg" and "err_exit" utilities are only used in ncep ops +# The "err_exit" utility is only used in ncep ops # when the "prod_util" module is loaded. #----------------------------------------------------------------------- @@ -134,10 +134,7 @@ export pgmout=${pgmout:-OUTPUT} # call utility script to create global ice blend data. #----------------------------------------------------------------------- -if test "$use_prod_util" = "true" ; then - msg="create blended ice data." - postmsg "$jlogfile" "$msg" -fi +echo "Create blended ice data." ${USHgfs}/emcsfc_ice_blend.sh rc=$? @@ -157,16 +154,10 @@ if ((rc != 0));then if test "$SENDCOM" = "YES" then if [ -s $BLENDED_ICE_FILE_m6hrs ]; then - msg="copy old ice blend file to current directory" - if test "$use_prod_util" = "true" ; then - postmsg "$jlogfile" "$msg" - fi + echo "Copy old ice blend file to current directory" cp $BLENDED_ICE_FILE_m6hrs $COMOUT/$BLENDED_ICE_FILE else - msg="FATAL ERROR: CURRENT AND 6-HR OLD ICE FILE MISSING" - if test "$use_prod_util" = "true" ; then - postmsg "$jlogfile" "$msg" - fi + echo "FATAL ERROR: CURRENT AND 6-HR OLD ICE FILE MISSING" if test "$use_prod_util" = "true" ; then err_exit else @@ -198,10 +189,7 @@ export GFS_LONSPERLAT_FILE=${LONSPERLAT:-$FIXgfs_am/global_lonsperlat.t${resolut export MODEL_SNOW_FILE=${FNSNOAJCAP:-${RUN}.${cycle}.snogrb_t${resolution}} export MODEL_SNOW_FILE_m6hrs=${FNSNOGJCAP:-${COMINgfs_m6hrs}/${RUN}.${cycle_m6hrs}.snogrb_t${resolution}} -if test "$use_prod_util" = "true" ; then - msg="create ${JCAP} snow data." - postmsg "$jlogfile" "$msg" -fi +echo "Create ${JCAP} snow data." ${USHgfs}/emcsfc_snow.sh rc=$? @@ -221,16 +209,10 @@ if ((rc != 0)); then if test "$SENDCOM" = "YES" then if [ -s $MODEL_SNOW_FILE_m6hrs ]; then - msg="COPY OLD ${JCAP} SNOW FILE TO CURRENT DIRECTORY" - if test "$use_prod_util" = "true" ; then - postmsg "$jlogfile" "$msg" - fi + echo "COPY OLD ${JCAP} SNOW FILE TO CURRENT DIRECTORY" cp $MODEL_SNOW_FILE_m6hrs $COMOUT/$MODEL_SNOW_FILE else - msg="FATAL ERROR: CURRENT AND 6-HR OLD ${JCAP} SNOW MISSING" - if test "$use_prod_util" = "true" ; then - postmsg "$jlogfile" "$msg" - fi + echo "FATAL ERROR: CURRENT AND 6-HR OLD ${JCAP} SNOW MISSING" if test "$use_prod_util" = "true" ; then err_exit else @@ -265,10 +247,7 @@ export GFS_LONSPERLAT_FILE=${LONSPERLAT_ENKF:-$FIXgfs_am/global_lonsperlat.t${re export MODEL_SNOW_FILE=${FNSNOAJCAP_ENKF:-${RUN}.${cycle}.snogrb_t${resolution}} export MODEL_SNOW_FILE_m6hrs=${FNSNOGJCAP_ENKF:-${COMINgfs_m6hrs}/${RUN}.${cycle_m6hrs}.snogrb_t${resolution}} -if test "$use_prod_util" = "true" ; then - msg="create enkf snow data." - postmsg "$jlogfile" "$msg" -fi +echo "Create enkf snow data." ${USHgfs}/emcsfc_snow.sh rc=$? @@ -282,16 +261,10 @@ if ((rc != 0)); then if test "$SENDCOM" = "YES" then if [ -s $MODEL_SNOW_FILE_m6hrs ]; then - msg="COPY OLD ENKF SNOW FILE TO CURRENT DIRECTORY" - if test "$use_prod_util" = "true" ; then - postmsg "$jlogfile" "$msg" - fi + echo "COPY OLD ENKF SNOW FILE TO CURRENT DIRECTORY" cp $MODEL_SNOW_FILE_m6hrs $COMOUT/$MODEL_SNOW_FILE else - msg="FATAL ERROR: CURRENT AND 6-HR OLD ENKF SNOW MISSING" - if test "$use_prod_util" = "true" ; then - postmsg "$jlogfile" "$msg" - fi + echo "FATAL ERROR: CURRENT AND 6-HR OLD ENKF SNOW MISSING" if test "$use_prod_util" = "true" ; then err_exit else diff --git a/ush/emcsfc_ice_blend.sh b/ush/emcsfc_ice_blend.sh index 9bceee83e..20ddbb373 100755 --- a/ush/emcsfc_ice_blend.sh +++ b/ush/emcsfc_ice_blend.sh @@ -45,8 +45,8 @@ if [[ "$VERBOSE" == YES ]]; then fi #----------------------------------------------------------------------- -# the "postmsg", "startmsg" and "prep_step" utilities -# are only used in ncep ops when the "prod_util" module is loaded. +# The "startmsg" and "prep_step" utilities are only +# used in ncep ops when the "prod_util" module is loaded. #----------------------------------------------------------------------- use_prod_util=`echo $UTILROOT` @@ -121,10 +121,7 @@ then grid173="0 0 0 0 0 0 0 0 4320 2160 0 0 89958000 42000 48 -89958000 359958000 83000 83000 0" $COPYGB2 -x -i3 -g "$grid173" ims.icec.grib2 ims.icec.5min.grib2 else - msg="WARNING in ${pgm}: IMS ice data missing. Can not run program." - if test "$use_prod_util" = "true" ; then - postmsg "$jlogfile" "$msg" - fi + echo "WARNING in ${pgm}: IMS ice data missing. Can not run program." exit 3 fi @@ -135,10 +132,7 @@ fi if [ ! -f ${FIVE_MIN_ICE_FILE} ] then - msg="WARNING in ${pgm}: MMAB ice data missing. Can not run program." - if test "$use_prod_util" = "true" ; then - postmsg "$jlogfile" "$msg" - fi + echo "WARNING in ${pgm}: MMAB ice data missing. Can not run program." exit 5 fi @@ -177,20 +171,14 @@ rc=$? if (( rc != 0 )) then - msg="WARNING: ${pgm} completed abnormally." - if test "$use_prod_util" = "true" ; then - postmsg "$jlogfile" "$msg" - fi + echo "WARNING: ${pgm} completed abnormally." exit $rc else $WGRIB2 -set_int 3 51 42000 ${BLENDED_ICE_FILE} -grib ${BLENDED_ICE_FILE}.corner $CNVGRIB -g21 ${BLENDED_ICE_FILE}.corner ${BLENDED_ICE_FILE}.bitmap rm $BLENDED_ICE_FILE $COPYGB -M "#1.57" -x ${BLENDED_ICE_FILE}.bitmap $BLENDED_ICE_FILE - msg="${pgm} completed normally." - if test "$use_prod_util" = "true" ; then - postmsg "$jlogfile" "$msg" - fi + echo "${pgm} completed normally." if [ "$SENDCOM" = "YES" ] ; then cp $BLENDED_ICE_FILE $COMOUT fi diff --git a/ush/emcsfc_snow.sh b/ush/emcsfc_snow.sh index 7c4c6c429..f19f01a40 100755 --- a/ush/emcsfc_snow.sh +++ b/ush/emcsfc_snow.sh @@ -52,12 +52,10 @@ if [[ "$VERBOSE" == YES ]]; then fi #----------------------------------------------------------------------- -# the "postmsg", "startmsg" and "prep_step" utilities -# are only used in ncep ops when the "prod_util" module is loaded. +# The "startmsg" and "prep_step" utilities are only +# used in ncep ops when the "prod_util" module is loaded. #----------------------------------------------------------------------- -jlogfile=${jlogfile:-"jlogfile"} - use_prod_util=`echo $UTILROOT` if ((${#use_prod_util} != 0)); then use_prod_util="true" @@ -146,10 +144,7 @@ $WGRIB2 ${IMS_FILE} rc1=$? if ((rc1 != 0));then - msg="WARNING: ${pgm} detects corrupt IMS data. Can not run." - if test "$use_prod_util" = "true" ; then - postmsg "$jlogfile" "$msg" - fi + echo "WARNING: ${pgm} detects corrupt IMS data. Can not run." exit $rc1 fi @@ -221,16 +216,10 @@ eval $SNOW2MDLEXEC >> $pgmout 2> errfile rc2=$? if ((rc2!= 0));then - msg="WARNING: ${pgm} completed abnormally." - if test "$use_prod_util" = "true" ; then - postmsg "$jlogfile" "$msg" - fi + echo "WARNING: ${pgm} completed abnormally." exit $rc2 else - msg="${pgm} completed normally." - if test "$use_prod_util" = "true" ; then - postmsg "$jlogfile" "$msg" - fi + echo "${pgm} completed normally." if test "$SENDCOM" = "YES" then cp $MODEL_SNOW_FILE $COMOUT From 3465d875ef85528faa3c3ae9b0ceeb09cdbf10d5 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 13 Jan 2022 21:23:20 +0000 Subject: [PATCH 35/57] Update library version defaults to match gfs v16.2. Remove load of envvar and replace with 'module reset'. Fixes #559. --- modulefiles/build.wcoss2.intel.lua | 7 ++----- sorc/machine-setup.sh | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/modulefiles/build.wcoss2.intel.lua b/modulefiles/build.wcoss2.intel.lua index 8c5f0f0f5..f17d1bea4 100644 --- a/modulefiles/build.wcoss2.intel.lua +++ b/modulefiles/build.wcoss2.intel.lua @@ -2,16 +2,13 @@ help([[ Load environment to compile UFS_UTILS on WCOSS2 ]]) -envvar_ver=os.getenv("envvar_ver") or "1.0" -load(pathJoin("envvar", envvar_ver)) - cmake_ver=os.getenv("cmake_ver") or "3.20.2" load(pathJoin("cmake", cmake_ver)) PrgEnv_intel_ver=os.getenv("PrgEnv_intel_ver") or "8.1.0" load(pathJoin("PrgEnv-intel", PrgEnv_intel_ver)) -craype_ver=os.getenv("craype_ver") or "2.7.8" +craype_ver=os.getenv("craype_ver") or "2.7.10" load(pathJoin("craype", craype_ver)) intel_ver=os.getenv("intel_ver") or "19.1.3.304" @@ -59,7 +56,7 @@ load(pathJoin("sp", sp_ver)) ip_ver=os.getenv("ip_ver") or "3.3.3" load(pathJoin("ip", ip_ver)) -g2_ver=os.getenv("g2_ver") or "3.4.1" +g2_ver=os.getenv("g2_ver") or "3.4.5" load(pathJoin("g2", g2_ver)) wgrib2_ver=os.getenv("wgrib2_ver") or "2.0.8" diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh index 3bd60c6c7..26d40982f 100644 --- a/sorc/machine-setup.sh +++ b/sorc/machine-setup.sh @@ -29,7 +29,7 @@ if [[ -d /lfs3 ]] ; then module purge elif [[ -d /lfs/h1 ]] ; then target=wcoss2 - module purge + module reset elif [[ -d /scratch1 ]] ; then # We are on NOAA Hera if ( ! eval module help > /dev/null 2>&1 ) ; then From 320eb127482eba6a23305e2664b18147f969b629 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 20 Jan 2022 20:28:48 +0000 Subject: [PATCH 36/57] Update wcoss2 regression test scripts for new baseline update logic. Fixes #559. --- reg_tests/chgres_cube/driver.wcoss2.sh | 7 +++++++ reg_tests/global_cycle/driver.wcoss2.sh | 7 +++++++ reg_tests/grid_gen/driver.wcoss2.sh | 7 +++++++ reg_tests/ice_blend/driver.wcoss2.sh | 7 +++++++ reg_tests/snow2mdl/driver.wcoss2.sh | 7 +++++++ 5 files changed, 35 insertions(+) diff --git a/reg_tests/chgres_cube/driver.wcoss2.sh b/reg_tests/chgres_cube/driver.wcoss2.sh index ce817cd9e..c29ef8e38 100755 --- a/reg_tests/chgres_cube/driver.wcoss2.sh +++ b/reg_tests/chgres_cube/driver.wcoss2.sh @@ -43,6 +43,13 @@ QUEUE="${QUEUE:-dev}" # and baseline data for each test. #----------------------------------------------------------------------------- +export UPDATE_BASELINE="FALSE" +#export UPDATE_BASELINE="TRUE" + +if [ "$UPDATE_BASELINE" = "TRUE" ]; then + source ../get_hash.sh +fi + export HOMEufs=$PWD/../.. export HOMEreg=/lfs/h2/emc/global/noscrub/George.Gayno/ufs_utils.git/reg_tests/chgres_cube diff --git a/reg_tests/global_cycle/driver.wcoss2.sh b/reg_tests/global_cycle/driver.wcoss2.sh index 419daab05..0406180d5 100755 --- a/reg_tests/global_cycle/driver.wcoss2.sh +++ b/reg_tests/global_cycle/driver.wcoss2.sh @@ -36,6 +36,13 @@ QUEUE="${QUEUE:-dev}" # Should not have to change anything below. #----------------------------------------------------------------------------- +export UPDATE_BASELINE="FALSE" +#export UPDATE_BASELINE="TRUE" + +if [ "$UPDATE_BASELINE" = "TRUE" ]; then + source ../get_hash.sh +fi + DATA_DIR="${WORK_DIR}/reg-tests/global-cycle" export HOMEreg=/lfs/h2/emc/global/noscrub/George.Gayno/ufs_utils.git/reg_tests/global_cycle diff --git a/reg_tests/grid_gen/driver.wcoss2.sh b/reg_tests/grid_gen/driver.wcoss2.sh index 2f2bdeb2d..11617cc6a 100755 --- a/reg_tests/grid_gen/driver.wcoss2.sh +++ b/reg_tests/grid_gen/driver.wcoss2.sh @@ -38,6 +38,13 @@ PROJECT_CODE="${PROJECT_CODE:-GFS-DEV}" # Should not have to change anything below here. #----------------------------------------------------------------------------- +export UPDATE_BASELINE="FALSE" +#export UPDATE_BASELINE="TRUE" + +if [ "$UPDATE_BASELINE" = "TRUE" ]; then + source ../get_hash.sh +fi + export NCCMP=/lfs/h2/emc/global/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp LOG_FILE=consistency.log diff --git a/reg_tests/ice_blend/driver.wcoss2.sh b/reg_tests/ice_blend/driver.wcoss2.sh index fb0eb054c..eb6609928 100755 --- a/reg_tests/ice_blend/driver.wcoss2.sh +++ b/reg_tests/ice_blend/driver.wcoss2.sh @@ -43,6 +43,13 @@ export DATA="${DATA}/reg-tests/ice-blend" # Should not have to change anything below. #----------------------------------------------------------------------------- +export UPDATE_BASELINE="FALSE" +#export UPDATE_BASELINE="TRUE" + +if [ "$UPDATE_BASELINE" = "TRUE" ]; then + source ../get_hash.sh +fi + export HOMEreg=/lfs/h2/emc/global/noscrub/George.Gayno/ufs_utils.git/reg_tests/ice_blend export HOMEgfs=$PBS_O_WORKDIR/../.. diff --git a/reg_tests/snow2mdl/driver.wcoss2.sh b/reg_tests/snow2mdl/driver.wcoss2.sh index e0596bf4b..2b4af6a0d 100755 --- a/reg_tests/snow2mdl/driver.wcoss2.sh +++ b/reg_tests/snow2mdl/driver.wcoss2.sh @@ -43,6 +43,13 @@ export DATA="${DATA}/reg-tests/snow2mdl" # Should not have to change anything below. #----------------------------------------------------------------------------- +export UPDATE_BASELINE="FALSE" +#export UPDATE_BASELINE="TRUE" + +if [ "$UPDATE_BASELINE" = "TRUE" ]; then + source ../get_hash.sh +fi + export HOMEreg=/lfs/h2/emc/global/noscrub/George.Gayno/ufs_utils.git/reg_tests/snow2mdl export HOMEgfs=$PWD/../.. From 32956d361479b59e98b81dc79292ab95fccad5c5 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Wed, 23 Mar 2022 19:27:45 +0000 Subject: [PATCH 37/57] Increase memory for some chgres grib2 consistency tests. Fixes #559. --- reg_tests/chgres_cube/driver.wcoss2.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reg_tests/chgres_cube/driver.wcoss2.sh b/reg_tests/chgres_cube/driver.wcoss2.sh index c29ef8e38..30f78b64d 100755 --- a/reg_tests/chgres_cube/driver.wcoss2.sh +++ b/reg_tests/chgres_cube/driver.wcoss2.sh @@ -155,7 +155,7 @@ TEST9=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime LOG_FILE=consistency.log10 export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" TEST10=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N 3km.conus.hrrr.gfssdf.grib2.conus -l select=1:ncpus=6:ompthreads=1:mem=40GB $PWD/3km.conus.hrrr.gfssdf.grib2.sh) + -N 3km.conus.hrrr.gfssdf.grib2.conus -l select=1:ncpus=6:ompthreads=1:mem=50GB $PWD/3km.conus.hrrr.gfssdf.grib2.sh) #----------------------------------------------------------------------------- # Initialize CONUS 3-KM USING HRRR GRIB2 file WITH GSD PHYSICS AND SFC VARS FROM FILE. @@ -164,7 +164,7 @@ TEST10=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltim LOG_FILE=consistency.log11 export APRUN="mpiexec -n 12 -ppn 12 --cpu-bind core" TEST11=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N 3km.conus.hrrr.newsfc.grib2.conus -l select=1:ncpus=12:ompthreads=1:mem=50GB $PWD/3km.conus.hrrr.newsfc.grib2.sh) + -N 3km.conus.hrrr.newsfc.grib2.conus -l select=1:ncpus=12:ompthreads=1:mem=75GB $PWD/3km.conus.hrrr.newsfc.grib2.sh) #----------------------------------------------------------------------------- # Initialize CONUS 13-KM USING NAM GRIB2 file WITH GFS PHYSICS . From 2bce1a8e9317833cafcba16aa785bc4effb38727 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Mon, 25 Apr 2022 20:22:57 +0000 Subject: [PATCH 38/57] Minor updates to ./reg_tests/chgres_cube/driver.wcoss2.sn Fixes #559. --- reg_tests/chgres_cube/driver.wcoss2.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reg_tests/chgres_cube/driver.wcoss2.sh b/reg_tests/chgres_cube/driver.wcoss2.sh index 30f78b64d..58bd5e8a5 100755 --- a/reg_tests/chgres_cube/driver.wcoss2.sh +++ b/reg_tests/chgres_cube/driver.wcoss2.sh @@ -155,7 +155,7 @@ TEST9=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime LOG_FILE=consistency.log10 export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" TEST10=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N 3km.conus.hrrr.gfssdf.grib2.conus -l select=1:ncpus=6:ompthreads=1:mem=50GB $PWD/3km.conus.hrrr.gfssdf.grib2.sh) + -N 3km.conus.hrrr.gfssdf.grib2.conus -l select=1:ncpus=6:ompthreads=1:mem=75GB $PWD/3km.conus.hrrr.gfssdf.grib2.sh) #----------------------------------------------------------------------------- # Initialize CONUS 3-KM USING HRRR GRIB2 file WITH GSD PHYSICS AND SFC VARS FROM FILE. @@ -221,7 +221,7 @@ qsub -V -o ${LOG_FILE} -e ${LOG_FILE} -q $QUEUE -A $PROJECT_CODE -l walltime=00: -W depend=afterok:$TEST1:$TEST2:$TEST3:$TEST4:$TEST5:$TEST6:$TEST7:$TEST8:$TEST9:$TEST10:$TEST11:$TEST12:$TEST13:$TEST14:$TEST15:$TEST16 << EOF #!/bin/bash cd ${this_dir} -grep -a '<<<' ${LOG_FILE}?? > $SUM_FILE +grep -a '<<<' ${LOG_FILE}?? | grep -v echo > $SUM_FILE EOF exit 0 From 99cda46fbe8ead1a4fa1e6f68668c2381f7c61e3 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 28 Apr 2022 18:26:27 +0000 Subject: [PATCH 39/57] Minor update to build script. Fixes #559. --- build_all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_all.sh b/build_all.sh index 0a8de2d7c..ee00be9e2 100755 --- a/build_all.sh +++ b/build_all.sh @@ -30,7 +30,7 @@ fi # access the EMC ftp site, so turn off the build (-DBUILD_TESTING=OFF) of the units tests accordingly. # Those with access to the EMC ftp site are: Orion, Hera, WCOSS-Cray, WCOSS-Dell. -if [[ "$target" == "hera" || "$target" == "orion" || "$target" == "wcoss_cray" || "$target" == "wcoss_dell_p3" ]]; then +if [[ "$target" == "hera" || "$target" == "orion" || "$target" == "wcoss_cray" || "$target" == "wcoss_dell_p3" || "$target" == "wcoss2" ]]; then CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=../ -DEMC_EXEC_DIR=ON -DBUILD_TESTING=OFF" #CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=../ -DEMC_EXEC_DIR=ON -DBUILD_TESTING=ON" #CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=../ -DEMC_EXEC_DIR=ON -DENABLE_DOCS=ON -DBUILD_TESTING=ON" From 263e786e4bfb0e49cbfff8763bed993e5bac9dcc Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 12 May 2022 13:56:22 +0000 Subject: [PATCH 40/57] Remove load of wgrib2 from build module. Fixes #559. --- modulefiles/build.wcoss2.intel.lua | 3 --- 1 file changed, 3 deletions(-) diff --git a/modulefiles/build.wcoss2.intel.lua b/modulefiles/build.wcoss2.intel.lua index f17d1bea4..9f1fc1008 100644 --- a/modulefiles/build.wcoss2.intel.lua +++ b/modulefiles/build.wcoss2.intel.lua @@ -59,9 +59,6 @@ load(pathJoin("ip", ip_ver)) g2_ver=os.getenv("g2_ver") or "3.4.5" load(pathJoin("g2", g2_ver)) -wgrib2_ver=os.getenv("wgrib2_ver") or "2.0.8" -load(pathJoin("wgrib2", wgrib2_ver)) - -- for mpiexec command cray_pals_ver=os.getenv("cray_pals_ver") or "1.0.12" load(pathJoin("cray-pals", cray_pals_ver)) From 8b893bbdf7c04bf91f40f9f912aefbc121d3f1a9 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 12 May 2022 15:15:36 +0000 Subject: [PATCH 41/57] Minor script updates. Fixes #559. --- reg_tests/global_cycle/driver.wcoss2.sh | 2 +- reg_tests/grid_gen/driver.wcoss2.sh | 2 +- reg_tests/ice_blend/driver.wcoss2.sh | 1 + reg_tests/snow2mdl/driver.wcoss2.sh | 1 + util/gdas_init/config | 12 ++++++------ 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/reg_tests/global_cycle/driver.wcoss2.sh b/reg_tests/global_cycle/driver.wcoss2.sh index 0406180d5..2b7512ae3 100755 --- a/reg_tests/global_cycle/driver.wcoss2.sh +++ b/reg_tests/global_cycle/driver.wcoss2.sh @@ -80,7 +80,7 @@ qsub -V -o ${LOG_FILE} -e ${LOG_FILE} -q $QUEUE -A $PROJECT_CODE -l walltime=00: -N cycle_summary -l select=1:ncpus=1:mem=100MB -W depend=afterok:$TEST1:$TEST2:$TEST3 << EOF #!/bin/bash cd $reg_dir -grep -a '<<<' ${LOG_FILE}?? > summary.log +grep -a '<<<' ${LOG_FILE}?? | grep -v echo > summary.log EOF exit diff --git a/reg_tests/grid_gen/driver.wcoss2.sh b/reg_tests/grid_gen/driver.wcoss2.sh index 11617cc6a..b5948b290 100755 --- a/reg_tests/grid_gen/driver.wcoss2.sh +++ b/reg_tests/grid_gen/driver.wcoss2.sh @@ -107,5 +107,5 @@ qsub -V -o ${LOG_FILE} -e ${LOG_FILE} -q $QUEUE -A $PROJECT_CODE -l walltime=00: -N grid_summary -l select=1:ncpus=1:mem=100MB -W depend=afterok:$TEST5 << EOF #!/bin/bash cd ${this_dir} -grep -a '<<<' $LOG_FILE > $SUM_FILE +grep -a '<<<' $LOG_FILE | grep -v echo > $SUM_FILE EOF diff --git a/reg_tests/ice_blend/driver.wcoss2.sh b/reg_tests/ice_blend/driver.wcoss2.sh index eb6609928..cf85af081 100755 --- a/reg_tests/ice_blend/driver.wcoss2.sh +++ b/reg_tests/ice_blend/driver.wcoss2.sh @@ -32,6 +32,7 @@ source ../../sorc/machine-setup.sh > /dev/null 2>&1 module use ../../modulefiles module load build.$target.intel module load grib_util/1.2.3 +module load wgrib2/2.0.8 module list set -x diff --git a/reg_tests/snow2mdl/driver.wcoss2.sh b/reg_tests/snow2mdl/driver.wcoss2.sh index 2b4af6a0d..ccaf1067b 100755 --- a/reg_tests/snow2mdl/driver.wcoss2.sh +++ b/reg_tests/snow2mdl/driver.wcoss2.sh @@ -32,6 +32,7 @@ source ../../sorc/machine-setup.sh > /dev/null 2>&1 module use ../../modulefiles module load build.$target.intel module load grib_util/1.2.2 +module load wgrib2/2.0.8 module list set -x diff --git a/util/gdas_init/config b/util/gdas_init/config index d31108fac..5ca148541 100644 --- a/util/gdas_init/config +++ b/util/gdas_init/config @@ -48,16 +48,16 @@ #----------------------------------------------------------- EXTRACT_DIR=/lfs/h2/emc/stmp/$USER/gdas.init/input -EXTRACT_DATA=yes +EXTRACT_DATA=no -RUN_CHGRES=no +RUN_CHGRES=yes -yy=2021 -mm=03 -dd=21 +yy=2022 +mm=05 +dd=06 hh=06 -use_v16retro=yes +use_v16retro=no LEVS=65 From 311c39a22cfe78feb7e9ab5af3f17169b25fa59b Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 12 May 2022 18:03:02 +0000 Subject: [PATCH 42/57] Update ./reg_tests/rt.sh for wcoss2. Fixes #559. --- reg_tests/rt.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/reg_tests/rt.sh b/reg_tests/rt.sh index dd25f276a..6e34d3bd4 100755 --- a/reg_tests/rt.sh +++ b/reg_tests/rt.sh @@ -96,6 +96,8 @@ for dir in snow2mdl ice_blend; do sbatch -A ${PROJECT_CODE} ./driver.$target.sh elif [[ $target == "wcoss_dell_p3" ]] || [[ $target == "wcoss_cray" ]]; then cat ./driver.$target.sh | bsub -P ${PROJECT_CODE} + elif [[ $target == "wcoss2" ]] ; then + qsub ./driver.$target.sh fi # Wait for job to complete @@ -115,8 +117,8 @@ done echo "Commit hash: ${current_hash}" >> ${WORK_DIR}/reg_test_results.txt echo "" >> ${WORK_DIR}/reg_test_results.txt +success=true for dir in chgres_cube grid_gen global_cycle ice_blend snow2mdl; do - success=true if grep -qi "FAILED" ${dir}/summary.log; then success=false echo "${dir} consistency tests FAILED" >> ${WORK_DIR}/reg_test_results.txt From 4339a58005b5f16016f2973df094aa3310a1c7ca Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Tue, 17 May 2022 19:47:17 +0000 Subject: [PATCH 43/57] Update rt.sh to check for which wcoss2 is production and development. Fixes #559. --- reg_tests/rt.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/reg_tests/rt.sh b/reg_tests/rt.sh index 6e34d3bd4..68de40d59 100755 --- a/reg_tests/rt.sh +++ b/reg_tests/rt.sh @@ -49,6 +49,13 @@ if [[ $target == "wcoss_dell_p3" ]] || [[ $target == "wcoss_cray" ]]; then if [[ "${this_letter}" == "${prod_letter}" ]]; then exit 0 fi +elif [[ $target == "wcoss2" ]]; then + this_machine=`cat /etc/cluster_name` + prod_machine=`grep primary /lfs/h1/ops/prod/config/prodmachinefile` + prod_machine=`echo ${prod_machine/primary:}` + if [[ "${this_machine}" == "${prod_machine}" ]]; then + exit 0 + fi fi # Set machine_id variable for running link_fixdirs From 14ad38c5ef6310840dd0f86af71ab0fe01fdecaf Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Mon, 23 May 2022 18:46:12 +0000 Subject: [PATCH 44/57] Update the snow2mdl consistency test script for the new test of afwa global data. Fixes #559. --- reg_tests/snow2mdl/driver.wcoss2.sh | 59 +++++++++++++++++++---------- 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/reg_tests/snow2mdl/driver.wcoss2.sh b/reg_tests/snow2mdl/driver.wcoss2.sh index ccaf1067b..833ae5e3d 100755 --- a/reg_tests/snow2mdl/driver.wcoss2.sh +++ b/reg_tests/snow2mdl/driver.wcoss2.sh @@ -2,32 +2,22 @@ #----------------------------------------------------------------------------- # -# Run snow2mdl consistency test on WCOSS2. +# Run snow2mdl consistency tests on WCOSS2. # -# Set $DATA to your working directory. Set the project code (PBS -A) -# and queue (PBS -q) as appropriate. +# Set $DATA_ROOT to your working directory. Set the project code +# and queue as appropriate. # -# Invoke the script as follows: qsub $script +# Invoke the script as follows: ./$script # # Log output is placed in consistency.log. A summary is # placed in summary.log # # The test fails when its output does not match the baseline file -# as determined by the 'cmp' command. The baseline file is +# as determined by the 'cmp' command. The baseline files are # stored in HOMEreg. # #----------------------------------------------------------------------------- -#PBS -l walltime=00:05:00 -#PBS -o consistency.log -#PBS -e consistency.log -#PBS -N s2m_regt -#PBS -q debug -#PBS -A GFS-DEV -#PBS -l select=1:ncpus=1:mem=2500MB - -cd $PBS_O_WORKDIR - source ../../sorc/machine-setup.sh > /dev/null 2>&1 module use ../../modulefiles module load build.$target.intel @@ -37,8 +27,11 @@ module list set -x -export DATA="${WORK_DIR:-/lfs/h2/emc/stmp/$LOGNAME}" -export DATA="${DATA}/reg-tests/snow2mdl" +export DATA_ROOT="${WORK_DIR:-/lfs/h2/emc/stmp/$LOGNAME}" +export DATA_ROOT="${DATA_ROOT}/reg-tests/snow2mdl" + +PROJECT_CODE=${PROJECT_CODE:-"GFS-DEV"} +QUEUE=${QUEUE:-"dev"} #----------------------------------------------------------------------------- # Should not have to change anything below. @@ -54,8 +47,36 @@ fi export HOMEreg=/lfs/h2/emc/global/noscrub/George.Gayno/ufs_utils.git/reg_tests/snow2mdl export HOMEgfs=$PWD/../.. -rm -fr $DATA +LOG_FILE=consistency.log +SUM_FILE=summary.log + +rm -fr $DATA_ROOT -./snow2mdl.sh +#----------------------------------------------------------------------------- +# Test GFS ops snow. +#----------------------------------------------------------------------------- +export DATA=$DATA_ROOT/test.ops +TEST1=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l select=1:ncpus=1:mem=2500MB \ + -N snow.ops -l walltime=00:03:00 $PWD/snow2mdl.ops.sh) + +#----------------------------------------------------------------------------- +# Test afwa global snow. +#----------------------------------------------------------------------------- + +export DATA=$DATA_ROOT/test.global +TEST2=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l select=1:ncpus=1:mem=2500MB \ + -N snow.global -l walltime=00:03:00 -W depend=afterok:$TEST1 $PWD/snow2mdl.global.sh) + +#----------------------------------------------------------------------------- +# Create summary log. +#----------------------------------------------------------------------------- + +this_dir=$PWD +qsub -V -o ${LOG_FILE} -e ${LOG_FILE} -q $QUEUE -A $PROJECT_CODE -l walltime=00:01:00 \ + -N snow_summary -l select=1:ncpus=1:mem=100MB -W depend=afterok:$TEST2 << EOF +#!/bin/bash +cd ${this_dir} +grep -a '<<<' $LOG_FILE | grep -v echo > $SUM_FILE +EOF exit 0 From cc6327daad9723bf882edbfeb5ec871ef5c2b055 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Wed, 25 May 2022 19:45:26 +0000 Subject: [PATCH 45/57] Increase memory for the global_cycle regression test. Fixes #559. --- reg_tests/global_cycle/driver.wcoss2.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reg_tests/global_cycle/driver.wcoss2.sh b/reg_tests/global_cycle/driver.wcoss2.sh index 2b7512ae3..c9fc286b7 100755 --- a/reg_tests/global_cycle/driver.wcoss2.sh +++ b/reg_tests/global_cycle/driver.wcoss2.sh @@ -69,12 +69,12 @@ TEST1=$(qsub -V -o ${LOG_FILE}01 -e ${LOG_FILE}01 -q $QUEUE -A $PROJECT_CODE -l export DATA="${DATA_DIR}/test2" export COMOUT=$DATA TEST2=$(qsub -V -o ${LOG_FILE}02 -e ${LOG_FILE}02 -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N c768.lndincsoil -l select=1:ncpus=12:mem=1GB $PWD/C768.lndincsoil.sh) + -N c768.lndincsoil -l select=1:ncpus=12:mem=8GB $PWD/C768.lndincsoil.sh) export DATA="${DATA_DIR}/test3" export COMOUT=$DATA TEST3=$(qsub -V -o ${LOG_FILE}03 -e ${LOG_FILE}03 -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N c768.lndincsnow -l select=1:ncpus=12:mem=1GB $PWD/C768.lndincsnow.sh) + -N c768.lndincsnow -l select=1:ncpus=12:mem=8GB $PWD/C768.lndincsnow.sh) qsub -V -o ${LOG_FILE} -e ${LOG_FILE} -q $QUEUE -A $PROJECT_CODE -l walltime=00:01:00 \ -N cycle_summary -l select=1:ncpus=1:mem=100MB -W depend=afterok:$TEST1:$TEST2:$TEST3 << EOF From 8b948ba29d27981ba30dbe576c5b93ea037e233d Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Fri, 3 Jun 2022 15:28:07 +0000 Subject: [PATCH 46/57] Update to esmf v8.2. Fixes #559. --- modulefiles/build.wcoss2.intel.lua | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/modulefiles/build.wcoss2.intel.lua b/modulefiles/build.wcoss2.intel.lua index 9f1fc1008..1f567cb21 100644 --- a/modulefiles/build.wcoss2.intel.lua +++ b/modulefiles/build.wcoss2.intel.lua @@ -8,7 +8,7 @@ load(pathJoin("cmake", cmake_ver)) PrgEnv_intel_ver=os.getenv("PrgEnv_intel_ver") or "8.1.0" load(pathJoin("PrgEnv-intel", PrgEnv_intel_ver)) -craype_ver=os.getenv("craype_ver") or "2.7.10" +craype_ver=os.getenv("craype_ver") or "2.7.13" load(pathJoin("craype", craype_ver)) intel_ver=os.getenv("intel_ver") or "19.1.3.304" @@ -17,6 +17,7 @@ load(pathJoin("intel", intel_ver)) cray_mpich_ver=os.getenv("cray_mpich_ver") or "8.1.7" load(pathJoin("cray-mpich", cray_mpich_ver)) + libjpeg_ver=os.getenv("libjpeg_ver") or "9c" load(pathJoin("libjpeg", libjpeg_ver)) @@ -32,9 +33,6 @@ load(pathJoin("hdf5", hdf5_ver)) netcdf_ver=os.getenv("netcdf_ver") or "4.7.4" load(pathJoin("netcdf", netcdf_ver)) -esmf_ver=os.getenv("esmf_ver") or "8.1.1" -load(pathJoin("esmf", esmf_ver)) - bacio_ver=os.getenv("bacio_ver") or "2.4.1" load(pathJoin("bacio", bacio_ver)) @@ -63,4 +61,10 @@ load(pathJoin("g2", g2_ver)) cray_pals_ver=os.getenv("cray_pals_ver") or "1.0.12" load(pathJoin("cray-pals", cray_pals_ver)) +setenv("HPC_OPT","/apps/ops/para/libs") +prepend_path("MODULEPATH", "/apps/ops/para/libs/modulefiles/compiler/intel/19.1.3.304") +prepend_path("MODULEPATH", "/apps/ops/para/libs/modulefiles/mpi/intel/19.1.3.304/cray-mpich/8.1.7") +esmf_ver=os.getenv("esmf_ver") or "8.2.1b04" +load(pathJoin("esmf", esmf_ver)) + whatis("Description: UFS_UTILS build environment") From e441ea09c141ab0b4756cb558646fd013161e3a6 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Fri, 3 Jun 2022 21:36:37 +0000 Subject: [PATCH 47/57] Update cpld_gridgen regression test for wcoss2. Fixes #559. --- modulefiles/build.wcoss2.intel.lua | 9 +++++++++ reg_tests/cpld_gridgen/rt.sh | 29 ++++++++++++++++++++++++----- ush/cpld_gridgen.sh | 4 +++- 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/modulefiles/build.wcoss2.intel.lua b/modulefiles/build.wcoss2.intel.lua index 1f567cb21..e3869cc8a 100644 --- a/modulefiles/build.wcoss2.intel.lua +++ b/modulefiles/build.wcoss2.intel.lua @@ -61,6 +61,15 @@ load(pathJoin("g2", g2_ver)) cray_pals_ver=os.getenv("cray_pals_ver") or "1.0.12" load(pathJoin("cray-pals", cray_pals_ver)) +udunits_ver=os.getenv("udunits_ver") or "2.2.28" +load(pathJoin("udunits", udunits_ver)) + +gsl_ver=os.getenv("gsl_ver") or "2.7" +load(pathJoin("gsl", gsl_ver)) + +nco_ver=os.getenv("nco_ver") or "4.9.7" +load(pathJoin("nco", nco_ver)) + setenv("HPC_OPT","/apps/ops/para/libs") prepend_path("MODULEPATH", "/apps/ops/para/libs/modulefiles/compiler/intel/19.1.3.304") prepend_path("MODULEPATH", "/apps/ops/para/libs/modulefiles/mpi/intel/19.1.3.304/cray-mpich/8.1.7") diff --git a/reg_tests/cpld_gridgen/rt.sh b/reg_tests/cpld_gridgen/rt.sh index ffda43358..824798376 100755 --- a/reg_tests/cpld_gridgen/rt.sh +++ b/reg_tests/cpld_gridgen/rt.sh @@ -54,7 +54,7 @@ check_results() { echo "....MISSING file" | tee -a $PATHRT/$REGRESSIONTEST_LOG test_status=FAIL else - nccmp -dmfqS $(basename ${file}) $file >>${PATHRT}/nccmp_${TEST_NAME}.log 2>&1 && d=$? || d=$? + $NCCMP -dmfqS $(basename ${file}) $file >>${PATHRT}/nccmp_${TEST_NAME}.log 2>&1 && d=$? || d=$? if [[ $d -ne 0 ]]; then echo "....NOT OK" | tee -a $PATHRT/$REGRESSIONTEST_LOG test_status=FAIL @@ -115,12 +115,22 @@ COMPILE_LOG=compile.log REGRESSIONTEST_LOG=RegressionTests_$target.$compiler.log rm -f fail_test* $COMPILE_LOG run_*.log nccmp_*.log summary.log -if [[ $target = hera ]]; then +if [[ $target = wcoss2 ]]; then + STMP=${STMP:-/lfs/h2/emc/stmp/$USER} + export MOM6_FIXDIR=/lfs/h2/emc/global/noscrub/George.Gayno/ufs_utils.git/reg_tests/cpld_gridgen/fix_mom6 + BASELINE_ROOT=/lfs/h2/emc/global/noscrub/George.Gayno/ufs_utils.git/reg_tests/cpld_gridgen/baseline_data + ACCOUNT=${ACCOUNT:-GFS-DEV} + export APRUN="mpiexec -n 1 -ppn 1 --cpu-bind core" + QUEUE=${QUEUE:-dev} + SBATCH_COMMAND="./cpld_gridgen.sh" + NCCMP=/lfs/h2/emc/global/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp +elif [[ $target = hera ]]; then STMP=${STMP:-/scratch1/NCEPDEV/stmp4/$USER} export MOM6_FIXDIR=/scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/cpld_gridgen/fix_mom6 BASELINE_ROOT=/scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/cpld_gridgen/baseline_data ACCOUNT=${ACCOUNT:-nems} QUEUE=${QUEUE:-batch} + NCCMP=nccmp PARTITION=hera SBATCH_COMMAND="./cpld_gridgen.sh" elif [[ $target = orion ]]; then @@ -129,6 +139,7 @@ elif [[ $target = orion ]]; then BASELINE_ROOT=/work/noaa/nems/role-nems/ufs_utils/reg_tests/cpld_gridgen/baseline_data ACCOUNT=${ACCOUNT:-nems} QUEUE=${QUEUE:-batch} + NCCMP=nccmp PARTITION=orion ulimit -s unlimited SBATCH_COMMAND="./cpld_gridgen.sh" @@ -138,6 +149,7 @@ elif [[ $target = jet ]]; then BASELINE_ROOT=/lfs4/HFIP/hfv3gfs/emc.nemspara/role.ufsutils/ufs_utils/reg_tests/cpld_gridgen/baseline_data ACCOUNT=${ACCOUNT:-h-nems} QUEUE=${QUEUE:-batch} + NCCMP=nccmp PARTITION=xjet ulimit -s unlimited SBATCH_COMMAND="./cpld_gridgen.sh" @@ -232,9 +244,16 @@ while read -r line || [ "$line" ]; do cp $PATHRT/parm/grid.nml.IN $RUNDIR cd $RUNDIR - sbatch --wait --ntasks-per-node=1 --nodes=1 --mem=4G -t 0:05:00 -A $ACCOUNT -q $QUEUE -J $TEST_NAME \ - --partition=$PARTITION -o $PATHRT/run_${TEST_NAME}.log -e $PATHRT/run_${TEST_NAME}.log \ - --wrap "$SBATCH_COMMAND $TEST_NAME" && d=$? || d=$? + if [[ $target = wcoss2 ]]; then + qsub -V -o $PATHRT/run_${TEST_NAME}.log -e $PATHRT/run_${TEST_NAME}.log -q $QUEUE -A $ACCOUNT \ + -Wblock=true -l walltime=00:05:00 -N $TEST_NAME -l select=1:ncpus=1:mem=8GB -v RESNAME=$TEST_NAME $SBATCH_COMMAND + d=0 + + else + sbatch --wait --ntasks-per-node=1 --nodes=1 --mem=4G -t 0:05:00 -A $ACCOUNT -q $QUEUE -J $TEST_NAME \ + --partition=$PARTITION -o $PATHRT/run_${TEST_NAME}.log -e $PATHRT/run_${TEST_NAME}.log \ + --wrap "$SBATCH_COMMAND $TEST_NAME" && d=$? || d=$? + fi if [[ d -ne 0 ]]; then error "Batch job for test $TEST_NAME did not finish successfully. Refer to run_${TEST_NAME}.log" diff --git a/ush/cpld_gridgen.sh b/ush/cpld_gridgen.sh index e0b2af0ed..bce186a62 100755 --- a/ush/cpld_gridgen.sh +++ b/ush/cpld_gridgen.sh @@ -18,7 +18,7 @@ function edit_namelist { -e "s/DO_POSTWGTS/$DO_POSTWGTS/g" } -export RESNAME=$1 +export RESNAME=${RESNAME:-$1} export DEBUG=.false. export MASKEDIT=.false. export DO_POSTWGTS=.false. @@ -95,6 +95,8 @@ if [ ! -d ${OUTDIR_PATH} ]; then mkdir -p ${OUTDIR_PATH} fi +cd ${OUTDIR_PATH} + edit_namelist < grid.nml.IN > grid.nml $APRUN ./cpld_gridgen From 8fe9166a51d7fddf5569b0e0c6eb2ef43425c14e Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Tue, 7 Jun 2022 13:56:34 +0000 Subject: [PATCH 48/57] Update ./link_fixdirs.sh for new location of fix files. Fixes #559. --- fix/link_fixdirs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fix/link_fixdirs.sh b/fix/link_fixdirs.sh index 32d2b8efa..28485ac4c 100755 --- a/fix/link_fixdirs.sh +++ b/fix/link_fixdirs.sh @@ -46,7 +46,7 @@ elif [ $machine = "jet" ]; then elif [ $machine = "orion" ]; then FIX_DIR="/work/noaa/global/glopara/fix" elif [ $machine = "wcoss2" ]; then - FIX_DIR="/lfs/h2/emc/global/noscrub/Kate.Friedman/glopara/FIX/fix" + FIX_DIR="/lfs/h2/emc/global/noscrub/kate.friedman/glopara/FIX/fix" elif [ $machine = "s4" ]; then FIX_DIR="/data/prod/glopara/fix" fi From 34f578c59e1b72da5754c9310c0a7d28e0533645 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Tue, 7 Jun 2022 15:31:33 +0000 Subject: [PATCH 49/57] Update ./reg_tests/cpld_gridgen/rt.sh to run regression tests on wcoss2. Fixes #559. --- reg_tests/cpld_gridgen/rt.sh | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/reg_tests/cpld_gridgen/rt.sh b/reg_tests/cpld_gridgen/rt.sh index 824798376..9c1d23366 100755 --- a/reg_tests/cpld_gridgen/rt.sh +++ b/reg_tests/cpld_gridgen/rt.sh @@ -245,18 +245,30 @@ while read -r line || [ "$line" ]; do cd $RUNDIR if [[ $target = wcoss2 ]]; then - qsub -V -o $PATHRT/run_${TEST_NAME}.log -e $PATHRT/run_${TEST_NAME}.log -q $QUEUE -A $ACCOUNT \ - -Wblock=true -l walltime=00:05:00 -N $TEST_NAME -l select=1:ncpus=1:mem=8GB -v RESNAME=$TEST_NAME $SBATCH_COMMAND - d=0 + +# rm -f $RUNDIR/bad.${TEST_NAME} + + TEST=$(qsub -V -o $PATHRT/run_${TEST_NAME}.log -e $PATHRT/run_${TEST_NAME}.log -q $QUEUE -A $ACCOUNT \ + -Wblock=true -l walltime=00:05:00 -N $TEST_NAME -l select=1:ncpus=1:mem=8GB -v RESNAME=$TEST_NAME $SBATCH_COMMAND) + +# qsub -o $PATHRT/run_${TEST_NAME}.log -e $PATHRT/run_${TEST_NAME}.log -q $QUEUE -A $ACCOUNT \ +# -Wblock=true -l walltime=00:01:00 -N chgres_summary -l select=1:ncpus=1:mem=100MB -W depend=afternotok:$TEST << EOF +#!/bin/bash +# touch $RUNDIR/bad.${TEST_NAME} +#EOF +# if [[ -f $RUNDIR/bad.${TEST_NAME} ]]; then +# error "Batch job for test $TEST_NAME did not finish successfully. Refer to run_${TEST_NAME}.log" +# fi else sbatch --wait --ntasks-per-node=1 --nodes=1 --mem=4G -t 0:05:00 -A $ACCOUNT -q $QUEUE -J $TEST_NAME \ --partition=$PARTITION -o $PATHRT/run_${TEST_NAME}.log -e $PATHRT/run_${TEST_NAME}.log \ --wrap "$SBATCH_COMMAND $TEST_NAME" && d=$? || d=$? - fi - if [[ d -ne 0 ]]; then - error "Batch job for test $TEST_NAME did not finish successfully. Refer to run_${TEST_NAME}.log" + if [[ d -ne 0 ]]; then + error "Batch job for test $TEST_NAME did not finish successfully. Refer to run_${TEST_NAME}.log" + fi + fi check_results From 9a13a6b2a03ae1766d35616fad5c6d08d60af18c Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Tue, 7 Jun 2022 15:37:49 +0000 Subject: [PATCH 50/57] Update ./reg_tests/rt.sh to run cpld_gridgen on wcoss2. Fixes #559. --- reg_tests/rt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reg_tests/rt.sh b/reg_tests/rt.sh index 7cd43f3d4..d85a00c19 100755 --- a/reg_tests/rt.sh +++ b/reg_tests/rt.sh @@ -74,7 +74,7 @@ cd fix cd ../reg_tests -if [[ $target == "orion" ]] || [[ $target == "jet" ]] || [[ $target == "hera" ]] ; then +if [[ $target == "orion" ]] || [[ $target == "jet" ]] || [[ $target == "hera" ]] || [[ $target == "wcoss2" ]] ; then cd cpld_gridgen export ACCOUNT=$PROJECT_CODE From 340a50f86070b6f7f5e4decc9f8fbff0febbf911 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Tue, 5 Jul 2022 19:44:21 +0000 Subject: [PATCH 51/57] Add new chgres_cube regression test to wcoss2 driver script. Fixes #559. --- reg_tests/chgres_cube/driver.wcoss2.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/reg_tests/chgres_cube/driver.wcoss2.sh b/reg_tests/chgres_cube/driver.wcoss2.sh index 58bd5e8a5..8c733853d 100755 --- a/reg_tests/chgres_cube/driver.wcoss2.sh +++ b/reg_tests/chgres_cube/driver.wcoss2.sh @@ -211,6 +211,15 @@ export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" TEST16=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ -N 25km.conus.gfs.pbgrib2.conus -l select=1:ncpus=6:ompthreads=1:mem=15GB $PWD/25km.conus.gfs.pbgrib2.sh) +#----------------------------------------------------------------------------- +# Initialize global C96 using GEFS GRIB2 files. +#----------------------------------------------------------------------------- + +LOG_FILE=consistency.log17 +export APRUN="mpiexec -n 6 -ppn 6 --cpu-bind core" +TEST17=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ + -N c96.gefs.grib2 -l select=1:ncpus=6:ompthreads=1:mem=15GB $PWD/c96.gefs.grib2.sh) + #----------------------------------------------------------------------------- # Create summary log. #----------------------------------------------------------------------------- @@ -218,7 +227,7 @@ TEST16=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltim LOG_FILE=consistency.log qsub -V -o ${LOG_FILE} -e ${LOG_FILE} -q $QUEUE -A $PROJECT_CODE -l walltime=00:01:00 \ -N chgres_summary -l select=1:ncpus=1:mem=100MB \ - -W depend=afterok:$TEST1:$TEST2:$TEST3:$TEST4:$TEST5:$TEST6:$TEST7:$TEST8:$TEST9:$TEST10:$TEST11:$TEST12:$TEST13:$TEST14:$TEST15:$TEST16 << EOF + -W depend=afterok:$TEST1:$TEST2:$TEST3:$TEST4:$TEST5:$TEST6:$TEST7:$TEST8:$TEST9:$TEST10:$TEST11:$TEST12:$TEST13:$TEST14:$TEST15:$TEST16:$TEST17 << EOF #!/bin/bash cd ${this_dir} grep -a '<<<' ${LOG_FILE}?? | grep -v echo > $SUM_FILE From af63b99ada7f0572acc09beb37319e73d57e27de Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Tue, 5 Jul 2022 19:58:53 +0000 Subject: [PATCH 52/57] Increase memory for global_cycle regression tests. Fixes #559. --- reg_tests/global_cycle/driver.wcoss2.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reg_tests/global_cycle/driver.wcoss2.sh b/reg_tests/global_cycle/driver.wcoss2.sh index c9fc286b7..970a3cab5 100755 --- a/reg_tests/global_cycle/driver.wcoss2.sh +++ b/reg_tests/global_cycle/driver.wcoss2.sh @@ -64,7 +64,7 @@ rm -f ${LOG_FILE}* export DATA="${DATA_DIR}/test1" export COMOUT=$DATA TEST1=$(qsub -V -o ${LOG_FILE}01 -e ${LOG_FILE}01 -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ - -N c768.fv3gfs -l select=1:ncpus=12:mem=8GB $PWD/C768.fv3gfs.sh) + -N c768.fv3gfs -l select=1:ncpus=12:mem=12GB $PWD/C768.fv3gfs.sh) export DATA="${DATA_DIR}/test2" export COMOUT=$DATA From eadc5cab18eb45f8ac3166e8fa55d1293c7f16f3 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Tue, 5 Jul 2022 20:07:06 +0000 Subject: [PATCH 53/57] Increase memory in ./util/vcoord_gen/run.wcoss2.sh. Fixes #559. --- util/vcoord_gen/run.wcoss2.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/vcoord_gen/run.wcoss2.sh b/util/vcoord_gen/run.wcoss2.sh index 0504db37c..64160f69c 100755 --- a/util/vcoord_gen/run.wcoss2.sh +++ b/util/vcoord_gen/run.wcoss2.sh @@ -6,7 +6,7 @@ #PBS -N vcoord #PBS -q debug #PBS -A GFS-DEV -#PBS -l select=1:ncpus=1:mem=20MB +#PBS -l select=1:ncpus=1:mem=100MB #------------------------------------------------------------------------------- # From 0268817dfbcbf6043416cd54306dd5700b35a9fe Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Wed, 6 Jul 2022 18:55:25 +0000 Subject: [PATCH 54/57] Boost requested memory for gdas init utility. Fixes #559. --- util/gdas_init/driver.wcoss2.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/gdas_init/driver.wcoss2.sh b/util/gdas_init/driver.wcoss2.sh index 2dcf65d7f..76a3bb329 100755 --- a/util/gdas_init/driver.wcoss2.sh +++ b/util/gdas_init/driver.wcoss2.sh @@ -29,7 +29,7 @@ if [ $EXTRACT_DATA == yes ]; then mkdir -p $EXTRACT_DIR QUEUE=dev_transfer - MEM=500MB + MEM=2GB WALLT="02:00:00" case $gfs_ver in @@ -125,7 +125,7 @@ if [ $RUN_CHGRES == yes ]; then WALLT="0:15:00" MEM=75GB if [ $CRES_HIRES == 'C768' ] ; then - MEM=175GB + MEM=250GB elif [ $CRES_HIRES == 'C1152' ] ; then MEM=350GB NODES=1 From 29e4dafaaa604c29ab32636b33c31954e3240503 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Wed, 6 Jul 2022 20:57:57 +0000 Subject: [PATCH 55/57] Minor fix to rt.sh. Fixes #559. --- reg_tests/rt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reg_tests/rt.sh b/reg_tests/rt.sh index d85a00c19..6ae214e1e 100755 --- a/reg_tests/rt.sh +++ b/reg_tests/rt.sh @@ -127,7 +127,7 @@ for dir in ice_blend; do elif [[ $target == "wcoss_dell_p3" ]] || [[ $target == "wcoss_cray" ]]; then cat ./driver.$target.sh | bsub -P ${PROJECT_CODE} elif [[ $target == "wcoss2" ]] ; then - qsub ./driver.$target.sh + qsub -v WORK_DIR ./driver.$target.sh fi # Wait for job to complete From b75f502026c70496af35c8feac7ac11e02047df4 Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 7 Jul 2022 13:06:41 +0000 Subject: [PATCH 56/57] Update README.md Fixes #559. --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f1fa48fd4..b9983b39f 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ https://ufs-community.github.io/UFS_UTILS/. Utility | Programmer(s) --------|---------- chgres_cube | George Gayno, Jeff Beck, Larissa Reames -cpld_gridgen | Denise Worthen, Minsuk Ji +cpld_gridgen | Denise Worthen emcsfc_ice_blend | George Gayno emcsfc_snow2mdl | George Gayno fre-nctools | GFDL progammer @@ -58,6 +58,14 @@ It also uses the following repositories: ## Installing +On Orion, Jet, Hera and WCOSS2, invoke the build script: + +``` +./build_all.sh +``` + +Otherwise, do: + ``` mkdir build cd build From b4b3570d05c77d40be564f2f9331ed37ddc976de Mon Sep 17 00:00:00 2001 From: "George.Gayno" Date: Thu, 7 Jul 2022 18:41:20 +0000 Subject: [PATCH 57/57] Increase requested wall clock time in grid_gen consistency test driver script. Remove 'cpld_gridgen' test from ./reg_tests/rt.sh as that program does not run on WCOSS2. Fixes #559. --- reg_tests/grid_gen/driver.wcoss2.sh | 12 ++++++------ reg_tests/rt.sh | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/reg_tests/grid_gen/driver.wcoss2.sh b/reg_tests/grid_gen/driver.wcoss2.sh index b5948b290..abf239c87 100755 --- a/reg_tests/grid_gen/driver.wcoss2.sh +++ b/reg_tests/grid_gen/driver.wcoss2.sh @@ -68,42 +68,42 @@ rm -fr $WORK_DIR # C96 uniform grid #----------------------------------------------------------------------------- -TEST1=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:08:00 \ +TEST1=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:10:00 \ -N c96.uniform -l select=1:ncpus=30:mem=40GB $PWD/c96.uniform.sh) #----------------------------------------------------------------------------- # C96 uniform grid using viirs vegetation data. #----------------------------------------------------------------------------- -TEST2=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:08:00 \ +TEST2=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:10:00 \ -N c96.viirs.vegt -l select=1:ncpus=30:mem=40GB -W depend=afterok:$TEST1 $PWD/c96.viirs.vegt.sh) #----------------------------------------------------------------------------- # gfdl regional grid #----------------------------------------------------------------------------- -TEST3=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:08:00 \ +TEST3=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:10:00 \ -N gfdl.regional -l select=1:ncpus=30:mem=40GB -W depend=afterok:$TEST2 $PWD/gfdl.regional.sh) #----------------------------------------------------------------------------- # esg regional grid #----------------------------------------------------------------------------- -TEST4=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:08:00 \ +TEST4=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:10:00 \ -N esg.regional -l select=1:ncpus=30:mem=40GB -W depend=afterok:$TEST3 $PWD/esg.regional.sh) #----------------------------------------------------------------------------- # Regional GSL gravity wave drag test. #----------------------------------------------------------------------------- -TEST5=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:08:00 \ +TEST5=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:10:00 \ -N rsg.gsl.gwd -l select=1:ncpus=30:mem=40GB -W depend=afterok:$TEST4 $PWD/regional.gsl.gwd.sh) #----------------------------------------------------------------------------- # Create summary log. #----------------------------------------------------------------------------- -qsub -V -o ${LOG_FILE} -e ${LOG_FILE} -q $QUEUE -A $PROJECT_CODE -l walltime=00:01:00 \ +qsub -V -o ${LOG_FILE} -e ${LOG_FILE} -q $QUEUE -A $PROJECT_CODE -l walltime=00:02:00 \ -N grid_summary -l select=1:ncpus=1:mem=100MB -W depend=afterok:$TEST5 << EOF #!/bin/bash cd ${this_dir} diff --git a/reg_tests/rt.sh b/reg_tests/rt.sh index 6ae214e1e..992e250d1 100755 --- a/reg_tests/rt.sh +++ b/reg_tests/rt.sh @@ -74,7 +74,8 @@ cd fix cd ../reg_tests -if [[ $target == "orion" ]] || [[ $target == "jet" ]] || [[ $target == "hera" ]] || [[ $target == "wcoss2" ]] ; then +#if [[ $target == "orion" ]] || [[ $target == "jet" ]] || [[ $target == "hera" ]] || [[ $target == "wcoss2" ]] ; then +if [[ $target == "orion" ]] || [[ $target == "jet" ]] || [[ $target == "hera" ]] ; then cd cpld_gridgen export ACCOUNT=$PROJECT_CODE