Add run_jedi_exe ctests, add example letkf yaml for orion (#223)

parm/atm/lgetkf/lgetkf.yaml

@@ -45,7 +45,8 @@ driver:
 local ensemble DA:
   solver: GETKF
   vertical localization:
-    fraction of retained variance: 0.975
+#   fraction of retained variance: 0.975  # use for scientific tests
+    fraction of retained variance: 0.750  # use for ctest
     lengthscale: 2.1
     lengthscale units: logp
   inflation:

parm/atm/variational/3dvar_dripcg.yaml

@@ -33,7 +33,7 @@ variational:
   minimizer:
     algorithm: DRIPCG
   iterations:
-  - ninner: 20
+  - ninner: 2
     gradient norm reduction: 1e-10
     test: on
     geometry:
@@ -50,7 +50,7 @@ variational:
       field metadata override: ./fv3jedi/gfs-restart.yaml
     diagnostics:
       departures: bkgmob
-  - ninner: 40
+  - ninner: 4
     gradient norm reduction: 1e-10
     test: on
     geometry:

test/CMakeLists.txt

@@ -140,6 +140,23 @@ add_test(NAME test_gdasapp_convert_ewok_yaml
 
 # high level tests that require the global-workflow
 if (WORKFLOW_TESTS)
+
+  # run_jedi_exe ctests do not require g-w.  place here due to longer run time
+  # test for ush/run_jedi_exe.py 3dhofx
+  add_test(NAME test_gdasapp_run_jedi_exe_3dhofx
+           COMMAND ${PROJECT_SOURCE_DIR}/test/run_jedi_exe_3dhofx.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
+           WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
+
+  # test for ush/run_jedi_exe.py 3dvar
+  add_test(NAME test_gdasapp_run_jedi_exe_3dvar
+           COMMAND ${PROJECT_SOURCE_DIR}/test/run_jedi_exe_3dvar.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
+           WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
+
+  # test for ush/run_jedi_exe.py letkf
+  add_test(NAME test_gdasapp_run_jedi_exe_letkf
+           COMMAND ${PROJECT_SOURCE_DIR}/test/run_jedi_exe_letkf.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
+           WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
+
   # test for creating an experiment directory within the global-workflow
   add_test(NAME test_gdasapp_setup_cycled_exp
            COMMAND ${PROJECT_SOURCE_DIR}/test/setup_workflow_exp.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}

test/run_jedi_exe_3dhofx.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+set -x
+
+bindir=$1
+srcdir=$2
+
+if [[ -d /scratch1 ]] ; then
+    machine="hera"
+elif [[ -d /work ]] ; then
+    machine="orion"
+else
+    echo "UNSUPPORTED MACHINE.  ABORT"
+    exit 99
+fi
+
+mkdir -p ${bindir}/test/testoutput/gdas_single_test_hofx3d
+cd ${bindir}/test/testoutput/gdas_single_test_hofx3d
+
+cat > ./3dhofx_example.yaml << EOF
+working directory: ./
+GDASApp home: ${srcdir}
+GDASApp mode: hofx
+template: ${srcdir}/parm/atm/hofx/hofx_nomodel.yaml
+config:
+  obs_yaml_dir: ${srcdir}/parm/atm/obs/config
+  executable: ${bindir}/bin/fv3jedi_hofx_nomodel.x
+  obs_list: ${srcdir}/parm/atm/obs/lists/gdas_prototype_3d.yaml
+  gdas_fix_root: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix
+  atm: true
+  layout_x: 1
+  layout_y: 1
+  atm_window_length: PT6H
+  valid_time: 2021-08-01T00:00:00Z
+  dump: gdas
+  case: C768
+  levs: 128
+  interp_method: barycentric
+job options:
+  machine: ${machine}
+  account: da-cpu
+  queue: debug
+  partition: hera
+  walltime: '30:00'
+  ntasks: 6
+  ntasks-per-node: 2
+  modulepath: ${srcdir}/modulefiles
+EOF
+
+rm stdout.txt
+${srcdir}/ush/run_jedi_exe.py -c ./3dhofx_example.yaml > stdout.txt
+rc=$?
+if [ $rc -ne 0 ]; then
+    exit $rc
+fi
+
+jobid=$(grep "Submitted" stdout.txt | awk -F' ' '{print $4}')
+echo "jobid is $jobid"
+
+nloop=100
+n=1
+while [ $n -le $nloop ]; do
+    if [ -s GDASApp.o$jobid ]; then
+	break
+    fi
+    sleep 10
+    n=$((n+1))
+done
+
+rc=1
+n=1
+while [ $n -le $nloop ]; do
+    count=$(cat GDASApp.o$jobid | grep "OOPS_STATS Run end" | wc -l)
+    echo "n = $n   count = $count"
+    if [ $count -gt 0 ]; then
+	rc=0
+	break
+    fi
+    sleep 10
+    n=$((n+1))
+done
+
+exit $rc

test/run_jedi_exe_3dvar.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+set -x
+
+bindir=$1
+srcdir=$2
+
+if [[ -d /scratch1 ]] ; then
+    machine="hera"
+elif [[ -d /work ]] ; then
+    machine="orion"
+else
+    echo "UNSUPPORTED MACHINE.  ABORT"
+    exit 99
+fi
+
+mkdir -p ${bindir}/test/testoutput/gdas_single_test_3dvar
+cd ${bindir}/test/testoutput/gdas_single_test_3dvar
+
+cat > ./3dvar_example.yaml << EOF
+working directory: ./
+GDASApp home: ${srcdir}
+GDASApp mode: variational
+template: ${srcdir}/parm/atm/variational/3dvar_dripcg.yaml
+config:
+  berror_yaml: ${srcdir}/parm/atm/berror/staticb_gsibec.yaml
+  obs_dir: obs
+  diag_dir: diags
+  crtm_coeff_dir: crtm
+  bias_in_dir: obs
+  bias_out_dir: bc
+  obs_yaml_dir: ${srcdir}/parm/atm/obs/config
+  executable: ${bindir}/bin/fv3jedi_var.x
+  obs_list: ${srcdir}/parm/atm/obs/lists/gdas_prototype_3d.yaml
+  gdas_fix_root: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix
+  atm: true
+  layout_x: 1
+  layout_y: 1
+  atm_window_length: PT6H
+  valid_time: 2021-12-21T06:00:00Z
+  dump: gdas
+  case: C96
+  case_anl: C96
+  staticb_type: gsibec
+  dohybvar: false
+  levs: 128
+  nmem: 10
+  interp_method: barycentric
+job options:
+  machine: ${machine}
+  account: da-cpu
+  queue: debug
+  partition: hera
+  walltime: '30:00'
+  ntasks: 6
+  modulepath: ${srcdir}/modulefiles
+EOF
+
+rm stdout.txt
+${srcdir}/ush/run_jedi_exe.py -c ./3dvar_example.yaml > stdout.txt
+rc=$?
+if [ $rc -ne 0 ]; then
+    exit $rc
+fi
+
+jobid=$(grep "Submitted" stdout.txt | awk -F' ' '{print $4}')
+echo "jobid is $jobid"
+
+nloop=100
+n=1
+while [ $n -le $nloop ]; do
+    if [ -s GDASApp.o$jobid ]; then
+	break
+    fi
+    sleep 10
+    n=$((n+1))
+done
+
+rc=1
+n=1
+while [ $n -le $nloop ]; do
+    count=$(cat GDASApp.o$jobid | grep "OOPS_STATS Run end" | wc -l)
+    echo "n = $n   count = $count"
+    if [ $count -gt 0 ]; then
+	rc=0
+	break
+    fi
+    sleep 10
+    n=$((n+1))
+done
+
+exit $rc

test/run_jedi_exe_letkf.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+set -x
+
+bindir=$1
+srcdir=$2
+
+if [[ -d /scratch1 ]] ; then
+    machine="hera"
+elif [[ -d /work ]] ; then
+    machine="orion"
+else
+    echo "UNSUPPORTED MACHINE.  ABORT"
+    exit 99
+fi
+
+if [ "$machine" = "hera" ] ; then
+    cominges="/scratch1/NCEPDEV/da/Russ.Treadon/GDASApp/cases"
+elif [ "$machine" = "orion" ]; then
+    cominges="/work2/noaa/da/rtreadon/GDASApp/cases"
+fi
+
+mkdir -p ${bindir}/test/testoutput/gdas_single_test_letkf
+cd ${bindir}/test/testoutput/gdas_single_test_letkf
+
+cat > ./letkf_example.yaml << EOF
+working directory: ./
+GDASApp home: ${srcdir}
+GDASApp mode: letkf
+template: ${srcdir}/parm/atm/lgetkf/lgetkf.yaml
+config:
+  obs_dir: obs
+  diag_dir: diags
+  crtm_coeff_dir: crtm
+  bias_in_dir: obs
+  bias_out_dir: bc
+  obs_yaml_dir: ${srcdir}/parm/atm/obs/config
+  executable: ${bindir}/bin/fv3jedi_letkf.x
+  obs_list: ${srcdir}/parm/atm/obs/lists/lgetkf_prototype.yaml
+  gdas_fix_root: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix
+  atm: true
+  layout_x: 1
+  layout_y: 1
+  atm_window_length: PT6H
+  valid_time: 2021-12-21T06:00:00Z
+  dump: gdas
+  case: C48
+  case_anl: C48
+  staticb_type: gsibec
+  dohybvar: no
+  levs: 128
+  nmem: 5
+  comin_ges: ${cominges}
+  interp_method: barycentric
+job options:
+  machine: ${machine}
+  account: da-cpu
+  queue: debug
+  partition: hera
+  walltime: '30:00'
+  ntasks: 6
+  modulepath: ${srcdir}/modulefiles
+EOF
+
+rm stdout.txt
+${srcdir}/ush/run_jedi_exe.py -c ./letkf_example.yaml > stdout.txt
+rc=$?
+if [ $rc -ne 0 ]; then
+    exit $rc
+fi
+
+jobid=$(grep "Submitted" stdout.txt | awk -F' ' '{print $4}')
+echo "jobid is $jobid"
+
+nloop=100
+n=1
+while [ $n -le $nloop ]; do
+    if [ -s GDASApp.o$jobid ]; then
+	break
+    fi
+    sleep 10
+    n=$((n+1))
+done
+
+rc=1
+n=1
+while [ $n -le $nloop ]; do
+    count=$(cat GDASApp.o$jobid | grep "OOPS_STATS Run end" | wc -l)
+    echo "n = $n   count = $count"
+    if [ $count -gt 0 ]; then
+	rc=0
+	break
+    fi
+    sleep 10
+    n=$((n+1))
+done
+
+exit $rc
+

ush/examples/run_jedi_exe/letkf_hera.yaml

@@ -25,7 +25,7 @@ config:
   dohybvar: no
   levs: 128
   nmem: 10
-  comin_ges: /scratch2/NCEPDEV/stmp1/Russ.Treadon/comrot/prufsda
+  comin_ges: /scratch1/NCEPDEV/da/Russ.Treadon/GDASApp/cases
   interp_method: barycentric
 job options:
   machine: hera

ush/examples/run_jedi_exe/letkf_orion.yaml

@@ -0,0 +1,37 @@
+working directory: /work2/noaa/stmp/cmartin/gdas_single_test_letkf
+GDASApp home: /work2/noaa/da/cmartin/GDASApp/work/GDASApp
+GDASApp mode: letkf
+template: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/lgetkf/lgetkf.yaml
+config:
+  obs_dir: obs
+  diag_dir: diags
+  crtm_coeff_dir: crtm
+  bias_in_dir: obs
+  bias_out_dir: bc
+  obs_yaml_dir: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/obs/config
+  executable: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/build/bin/fv3jedi_letkf.x
+  obs_list: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/obs/lists/lgetkf_prototype.yaml
+  gdas_fix_root: /work2/noaa/da/cmartin/GDASApp/fix
+  atm: true
+  layout_x: 1
+  layout_y: 1
+  atm_window_length: PT6H
+  valid_time: 2021-12-21T06:00:00Z
+  dump: gdas
+  case: C48
+  case_anl: C48
+  case_enkf: C48
+  staticb_type: gsibec
+  dohybvar: no
+  levs: 128
+  nmem: 10
+  comin_ges: /work2/noaa/da/rtreadon/GDASApp/cases
+  interp_method: barycentric
+job options:
+  machine: hera
+  account: da-cpu
+  queue: debug
+  partition: hera
+  walltime: '30:00'
+  ntasks: 6
+  modulepath: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/modulefiles