From 87cf706094e05849dbcc8cbe4c6fe49471b4ddfe Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Wed, 16 Aug 2023 09:31:40 -0400 Subject: [PATCH 01/13] Update Orion data stage location to new role-epic space --- ush/machine/orion.yaml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/ush/machine/orion.yaml b/ush/machine/orion.yaml index f203e3af46..3a3e35f631 100644 --- a/ush/machine/orion.yaml +++ b/ush/machine/orion.yaml @@ -2,10 +2,10 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 40 SCHED: slurm - TEST_CCPA_OBS_DIR: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/obs_data/ccpa/proc - TEST_MRMS_OBS_DIR: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/obs_data/mrms/proc - TEST_NDAS_OBS_DIR: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/obs_data/ndas/proc - DOMAIN_PREGEN_BASEDIR: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/FV3LAM_pregen + TEST_CCPA_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc + TEST_MRMS_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/mrms/proc + TEST_NDAS_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/ndas/proc + DOMAIN_PREGEN_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/FV3LAM_pregen PARTITION_DEFAULT: orion QUEUE_DEFAULT: batch PARTITION_FCST: orion @@ -21,17 +21,17 @@ platform: RUN_CMD_AQMLBC: srun --export=ALL -n ${NUMTS} SCHED_NATIVE_CMD: --export=NONE PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' - TEST_EXTRN_MDL_SOURCE_BASEDIR: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/input_model_data - TEST_AQM_INPUT_BASEDIR: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/aqm_data - TEST_PREGEN_BASEDIR: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/FV3LAM_pregen - TEST_VX_FCST_INPUT_BASEDIR: '{{ "/work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/output_data/fcst_" }}{{ "ens" if (global.NUM_ENS_MEMBERS > 0) else "det" }}{{ "/{{workflow.PREDEF_GRID_NAME}}" }}{% raw %}{% endraw %}' - FIXaer: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/fix/fix_aer - FIXgsi: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/fix/fix_gsi - FIXgsm: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/fix/fix_am - FIXlut: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/fix/fix_lut - FIXorg: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/fix/fix_orog - FIXsfc: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/fix/fix_sfc_climo - FIXshp: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/NaturalEarth + TEST_EXTRN_MDL_SOURCE_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data + TEST_AQM_INPUT_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/aqm_data + TEST_PREGEN_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/FV3LAM_pregen + TEST_VX_FCST_INPUT_BASEDIR: '{{ "/work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/output_data/fcst_" }}{{ "ens" if (global.NUM_ENS_MEMBERS > 0) else "det" }}{{ "/{{workflow.PREDEF_GRID_NAME}}" }}{% raw %}{% endraw %}' + FIXaer: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_aer + FIXgsi: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_gsi + FIXgsm: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_am + FIXlut: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_lut + FIXorg: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_orog + FIXsfc: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_sfc_climo + FIXshp: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/NaturalEarth EXTRN_MDL_DATA_STORES: aws nomads task_process_bufr: OBSPATH_TEMPLATE: '${OBSPATH}/${YYYYMMDD}.rap/rap' From 57605c317e81061ab3a3683691bbaa0c2f36c918 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Wed, 16 Aug 2023 09:36:01 -0400 Subject: [PATCH 02/13] modulefiles update for netcdf/4.9.2-based hpc-stacks on Tier-1 platofms --- modulefiles/build_cheyenne_gnu.lua | 8 +------- modulefiles/build_cheyenne_intel.lua | 9 +-------- modulefiles/build_gaea_intel.lua | 2 +- modulefiles/build_hera_gnu.lua | 6 +++--- modulefiles/build_hera_intel.lua | 4 ++-- modulefiles/build_jet_intel.lua | 2 +- modulefiles/build_orion_intel.lua | 4 ++-- modulefiles/srw_common.lua | 15 ++++++++------- 8 files changed, 19 insertions(+), 31 deletions(-) diff --git a/modulefiles/build_cheyenne_gnu.lua b/modulefiles/build_cheyenne_gnu.lua index 0504fccbb3..e9ed311712 100644 --- a/modulefiles/build_cheyenne_gnu.lua +++ b/modulefiles/build_cheyenne_gnu.lua @@ -14,19 +14,13 @@ setenv("MKLROOT", "/glade/u/apps/opt/intel/2022.1/mkl/latest") load(pathJoin("ncarcompilers", os.getenv("ncarcompilers_ver") or "0.5.0")) unload("netcdf") -prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/hpc-stack/gnu11.2.0/modulefiles/stack") +prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/hpc-stack/gnu11.2.0_ncdf492/modulefiles/stack") load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) load(pathJoin("hpc-gnu", os.getenv("hpc_gnu_ver") or "11.2.0")) load(pathJoin("hpc-mpt", os.getenv("hpc_mpt_ver") or "2.25")) load("srw_common") -load(pathJoin("g2", os.getenv("g2_ver") or "3.4.5")) -load(pathJoin("esmf", os.getenv("esmf_ver") or "8.3.0b09")) -load(pathJoin("netcdf", os.getenv("netcdf_ver") or "4.7.4")) -load(pathJoin("libpng", os.getenv("libpng_ver") or "1.6.37")) -load(pathJoin("pio", os.getenv("pio_ver") or "2.5.7")) -load(pathJoin("fms", os.getenv("fms_ver") or "2022.04")) load(pathJoin("openblas", os.getenv("openblas_ver") or "0.3.23")) unsetenv("MKLROOT") diff --git a/modulefiles/build_cheyenne_intel.lua b/modulefiles/build_cheyenne_intel.lua index ebbaa1d069..21bc7108ab 100644 --- a/modulefiles/build_cheyenne_intel.lua +++ b/modulefiles/build_cheyenne_intel.lua @@ -14,20 +14,13 @@ load(pathJoin("python", os.getenv("python_ver") or "3.7.9")) load(pathJoin("ncarcompilers", os.getenv("ncarcompilers_ver") or "0.5.0")) unload("netcdf") -prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/hpc-stack/intel2022.1/modulefiles/stack") +prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/hpc-stack/intel2022.1_ncdf492/modulefiles/stack") load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver") or "2022.1")) load(pathJoin("hpc-mpt", os.getenv("hpc_mpt_ver") or "2.25")) load("srw_common") -load(pathJoin("g2", os.getenv("g2_ver") or "3.4.5")) -load(pathJoin("esmf", os.getenv("esmf_ver") or "8.3.0b09")) -load(pathJoin("netcdf", os.getenv("netcdf_ver") or "4.7.4")) -load(pathJoin("libpng", os.getenv("libpng_ver") or "1.6.37")) -load(pathJoin("pio", os.getenv("pio_ver") or "2.5.7")) -load(pathJoin("fms", os.getenv("fms_ver") or "2022.04")) - setenv("CMAKE_C_COMPILER","mpicc") setenv("CMAKE_CXX_COMPILER","mpicpc") setenv("CMAKE_Fortran_COMPILER","mpif90") diff --git a/modulefiles/build_gaea_intel.lua b/modulefiles/build_gaea_intel.lua index bb156952e3..b840fa5a11 100644 --- a/modulefiles/build_gaea_intel.lua +++ b/modulefiles/build_gaea_intel.lua @@ -7,7 +7,7 @@ whatis([===[Loads libraries needed for building the UFS SRW App on Gaea ]===]) load(pathJoin("cmake", os.getenv("cmake_ver") or "3.20.1")) -prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2/modulefiles/stack") +prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2_ncdf492/modulefiles/stack") load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) load(pathJoin("hpc-intel-classic", os.getenv("hpc_intel_classic_ver") or "2022.0.2")) load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "7.7.20")) diff --git a/modulefiles/build_hera_gnu.lua b/modulefiles/build_hera_gnu.lua index 888a358549..2633fae842 100644 --- a/modulefiles/build_hera_gnu.lua +++ b/modulefiles/build_hera_gnu.lua @@ -13,7 +13,7 @@ load(pathJoin("cmake", os.getenv("cmake_ver") or "3.20.1")) gnu_ver=os.getenv("gnu_ver") or "9.2.0" load(pathJoin("gnu", gnu_ver)) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/gnu-9.2/modulefiles/stack") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/gnu-9.2_ncdf492/modulefiles/stack") load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) load(pathJoin("hpc-gnu", os.getenv("hpc-gnu_ver") or "9.2")) @@ -21,8 +21,8 @@ load(pathJoin("hpc-mpich", os.getenv("hpc-mpich_ver") or "3.3.2")) load("srw_common") -load(pathJoin("nccmp", os.getenv("nccmp_ver") or "1.8.9")) -load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3")) +load(pathJoin("nccmp", os.getenv("nccmp_ver") or "1.9.1.0")) +load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6")) load(pathJoin("openblas", os.getenv("openblas_ver") or "0.3.23")) unsetenv("MKLROOT") diff --git a/modulefiles/build_hera_intel.lua b/modulefiles/build_hera_intel.lua index 02af59a626..7ec884eeba 100644 --- a/modulefiles/build_hera_intel.lua +++ b/modulefiles/build_hera_intel.lua @@ -16,7 +16,7 @@ load(pathJoin("intel", intel_ver)) impi_ver=os.getenv("impi_ver") or "2022.1.2" load(pathJoin("impi", impi_ver)) -prepend_path("MODULEPATH","/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/modulefiles/stack") +prepend_path("MODULEPATH","/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2_ncdf492/modulefiles/stack") load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver") or "2022.1.2")) @@ -25,7 +25,7 @@ load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver") or "2022.1.2")) load("srw_common") load(pathJoin("nccmp", os.getenv("nccmp_ver") or "1.8.9.0")) -load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3")) +load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6")) setenv("CMAKE_C_COMPILER","mpiicc") setenv("CMAKE_CXX_COMPILER","mpiicpc") diff --git a/modulefiles/build_jet_intel.lua b/modulefiles/build_jet_intel.lua index 01c120daa3..4387a1ad33 100644 --- a/modulefiles/build_jet_intel.lua +++ b/modulefiles/build_jet_intel.lua @@ -10,7 +10,7 @@ load("sutils") load(pathJoin("cmake", os.getenv("cmake_ver") or "3.20.1")) -prepend_path("MODULEPATH","/mnt/lfs4/HFIP/hfv3gfs/role.epic/hpc-stack/libs/intel-2022.1.2/modulefiles/stack") +prepend_path("MODULEPATH","/mnt/lfs4/HFIP/hfv3gfs/role.epic/hpc-stack/libs/intel-2022.1.2_ncdf492/modulefiles/stack") load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver") or "2022.1.2")) load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver") or "2022.1.2")) diff --git a/modulefiles/build_orion_intel.lua b/modulefiles/build_orion_intel.lua index 79eb5c6487..87e35a0a35 100644 --- a/modulefiles/build_orion_intel.lua +++ b/modulefiles/build_orion_intel.lua @@ -11,7 +11,7 @@ load("noaatools") load(pathJoin("cmake", os.getenv("cmake_ver") or "3.22.1")) load(pathJoin("python", os.getenv("python_ver") or "3.9.2")) -prepend_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/orion/hpc-stack/intel-2022.1.2/modulefiles/stack") +prepend_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/orion/hpc-stack/intel-2022.1.2_ncdf492/modulefiles/stack") load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver") or "2022.1.2")) load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver") or "2022.1.2")) @@ -19,7 +19,7 @@ load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver") or "2022.1.2")) load("srw_common") load(pathJoin("nccmp", os.getenv("nccmp_ver") or "1.8.9.0")) -load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3")) +load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6")) setenv("CMAKE_C_COMPILER","mpiicc") setenv("CMAKE_CXX_COMPILER","mpiicpc") diff --git a/modulefiles/srw_common.lua b/modulefiles/srw_common.lua index 12b5122c18..782098c62d 100644 --- a/modulefiles/srw_common.lua +++ b/modulefiles/srw_common.lua @@ -1,12 +1,13 @@ load("jasper/2.0.25") load("zlib/1.2.11") -load_any("png/1.6.35", "libpng/1.6.37") -load_any("netcdf/4.7.4", "netcdf-c/4.7.4") -load_any("netcdf/4.7.4", "netcdf-fortran/4.5.4") -load_any("pio/2.5.7", "parallelio/2.5.2") -load_any("esmf/8.3.0b09", "esmf/8.2.0") -load("fms/2022.04") +load("libpng/1.6.37") + +load_any("netcdf/4.9.2", "netcdf-c/4.9.2") +load_any("netcdf/4.9.2", "netcdf-fortran/4.6.0") +load_any("pio/2.5.10", "parallelio/2.5.10") +load_any("esmf/8.4.2", "esmf/8.5.0") +load("fms/2023.01") load("bufr/11.7.0") load("bacio/2.4.1") @@ -19,7 +20,7 @@ load("w3emc/2.9.2") load_any("gftl-shared/v1.5.0", "gftl-shared/1.5.0") load_any("yafyaml/v0.5.1", "yafyaml/0.5.1") -load_any("mapl/2.22.0-esmf-8.3.0b09", "mapl/2.11.0-esmf-8.2.0") +load_any("mapl/2.35.2-esmf-8.4.2", "mapl/2.35.2-esmf-8.5.0") load("nemsio/2.5.4") load("sfcio/1.4.1") From e8875919abf698e837fd926516578eb93170977f Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Thu, 24 Aug 2023 19:53:18 -0400 Subject: [PATCH 03/13] Gaea modulefile update, use intel-classic-2023.1.0 --- modulefiles/build_gaea_intel.lua | 10 +++++----- modulefiles/wflow_gaea.lua | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/modulefiles/build_gaea_intel.lua b/modulefiles/build_gaea_intel.lua index b840fa5a11..c6410fdbc6 100644 --- a/modulefiles/build_gaea_intel.lua +++ b/modulefiles/build_gaea_intel.lua @@ -7,21 +7,21 @@ whatis([===[Loads libraries needed for building the UFS SRW App on Gaea ]===]) load(pathJoin("cmake", os.getenv("cmake_ver") or "3.20.1")) -prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2_ncdf492/modulefiles/stack") +prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2023.1.0/modulefiles/stack") load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) -load(pathJoin("hpc-intel-classic", os.getenv("hpc_intel_classic_ver") or "2022.0.2")) +load(pathJoin("hpc-intel-classic", os.getenv("hpc_intel_classic_ver") or "2023.1.0")) load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "7.7.20")) load("srw_common") -- Need at runtime load("alps") -local MKLROOT="/opt/intel/oneapi/mkl/2022.0.2/" +local MKLROOT="/opt/intel/oneapi/mkl/2023.1.0/" prepend_path("LD_LIBRARY_PATH",pathJoin(MKLROOT,"lib/intel64")) pushenv("MKLROOT", MKLROOT) -pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix/20230601") -pushenv("CRAYPE_LINK_TYPE","dynamic") +--pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix/20230601") +--pushenv("CRAYPE_LINK_TYPE","dynamic") setenv("CC","cc") setenv("FC","ftn") diff --git a/modulefiles/wflow_gaea.lua b/modulefiles/wflow_gaea.lua index 007e2b1fcd..9dd3632a07 100644 --- a/modulefiles/wflow_gaea.lua +++ b/modulefiles/wflow_gaea.lua @@ -14,7 +14,7 @@ load("rocoto") load("alps") pushenv("MKLROOT", "/opt/intel/oneapi/mkl/2022.0.2/") -pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix/20230601") +--pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix/20230601") setenv("PMI_NO_PREINITIALIZE","1") if mode() == "load" then From f327b3ee0d09a584dc5546b8f760c2a4311f7b1d Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Fri, 25 Aug 2023 10:19:45 -0400 Subject: [PATCH 04/13] increased task walltime request and maxtries=2 for getting external data from AWS --- parm/wflow/coldstart.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/parm/wflow/coldstart.yaml b/parm/wflow/coldstart.yaml index c1935f2160..6ee9c63380 100644 --- a/parm/wflow/coldstart.yaml +++ b/parm/wflow/coldstart.yaml @@ -22,6 +22,9 @@ default_task: &default_task task_get_extrn_ics: <<: *default_task command: '&LOAD_MODULES_RUN_TASK_FP; "get_extrn_ics" "&JOBSdir;/JREGIONAL_GET_EXTRN_MDL_FILES"' + attrs: + cycledefs: forecast + maxtries: '2' envars: <<: *default_vars ICS_OR_LBCS: ICS @@ -32,7 +35,7 @@ task_get_extrn_ics: partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' ppn: 1 queue: '&QUEUE_HPSS;' - walltime: 00:45:00 + walltime: 01:30:00 dependency: or: and: @@ -50,6 +53,9 @@ task_get_extrn_ics: task_get_extrn_lbcs: <<: *default_task command: '&LOAD_MODULES_RUN_TASK_FP; "get_extrn_lbcs" "&JOBSdir;/JREGIONAL_GET_EXTRN_MDL_FILES"' + attrs: + cycledefs: forecast + maxtries: '2' envars: <<: *default_vars ICS_OR_LBCS: LBCS @@ -60,7 +66,7 @@ task_get_extrn_lbcs: partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' ppn: 1 queue: '&QUEUE_HPSS;' - walltime: 00:45:00 + walltime: 01:30:00 dependency: or: and: From cfd2263b40b1333461618cabb9e063311e98d7c8 Mon Sep 17 00:00:00 2001 From: Natalie Perlin <68030316+natalie-perlin@users.noreply.github.com> Date: Thu, 31 Aug 2023 09:51:35 -0400 Subject: [PATCH 05/13] Update jet.yaml, use xjet only for forecast tasks Running a forecast on xjet only. Fails on other partitions. --- ush/machine/jet.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/machine/jet.yaml b/ush/machine/jet.yaml index a110f02614..bde54d8f7c 100644 --- a/ush/machine/jet.yaml +++ b/ush/machine/jet.yaml @@ -8,7 +8,7 @@ platform: DOMAIN_PREGEN_BASEDIR: /mnt/lfs4/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/FV3LAM_pregen PARTITION_DEFAULT: sjet,vjet,kjet,xjet QUEUE_DEFAULT: batch - PARTITION_FCST: sjet,vjet,kjet,xjet + PARTITION_FCST: xjet QUEUE_FCST: batch PARTITION_HPSS: service QUEUE_HPSS: batch From 61ab62049b25add30874b3456abb7ada97828a91 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Fri, 1 Sep 2023 21:59:54 -0400 Subject: [PATCH 06/13] srw_common.lua update to use for both hpc-stack and spack-stack --- modulefiles/srw_common.lua | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/modulefiles/srw_common.lua b/modulefiles/srw_common.lua index 782098c62d..bd91ae5f81 100644 --- a/modulefiles/srw_common.lua +++ b/modulefiles/srw_common.lua @@ -1,12 +1,11 @@ -load("jasper/2.0.25") -load("zlib/1.2.11") - +load_any("jasper/2.0.25","jasper/2.0.32") +load_any("zlib/1.2.11","zlib/1.2.13") load("libpng/1.6.37") load_any("netcdf/4.9.2", "netcdf-c/4.9.2") load_any("netcdf/4.9.2", "netcdf-fortran/4.6.0") -load_any("pio/2.5.10", "parallelio/2.5.10") -load_any("esmf/8.4.2", "esmf/8.5.0") +load_any("pio/2.5.10","parallelio/2.5.9") +load("esmf/8.4.2") load("fms/2023.01") load("bufr/11.7.0") @@ -20,14 +19,14 @@ load("w3emc/2.9.2") load_any("gftl-shared/v1.5.0", "gftl-shared/1.5.0") load_any("yafyaml/v0.5.1", "yafyaml/0.5.1") -load_any("mapl/2.35.2-esmf-8.4.2", "mapl/2.35.2-esmf-8.5.0") +load("mapl/2.35.2-esmf-8.4.2") load("nemsio/2.5.4") load("sfcio/1.4.1") load("sigio/2.3.2") load("w3nco/2.4.1") -load("wrf_io/1.2.0") +load_any("wrf_io/1.2.0","wrf-io/1.2.0") -load("ncdiag/1.1.1") +--load("ncdiag/1.1.1") load("ncio/1.1.2") load("wgrib2/2.0.8") From a1214894000334f3ab5386b28b8a8ce823804875 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Tue, 5 Sep 2023 12:08:13 -0400 Subject: [PATCH 07/13] update srw_common.lua, modulefile for Gaea C4: use intel/2023.1.0 compiler --- modulefiles/build_gaea_intel.lua | 11 ++++------- modulefiles/srw_common.lua | 13 ++++++------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/modulefiles/build_gaea_intel.lua b/modulefiles/build_gaea_intel.lua index c6410fdbc6..d8284ce9af 100644 --- a/modulefiles/build_gaea_intel.lua +++ b/modulefiles/build_gaea_intel.lua @@ -16,13 +16,6 @@ load("srw_common") -- Need at runtime load("alps") -local MKLROOT="/opt/intel/oneapi/mkl/2023.1.0/" -prepend_path("LD_LIBRARY_PATH",pathJoin(MKLROOT,"lib/intel64")) -pushenv("MKLROOT", MKLROOT) - ---pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix/20230601") ---pushenv("CRAYPE_LINK_TYPE","dynamic") - setenv("CC","cc") setenv("FC","ftn") setenv("CXX","CC") @@ -31,3 +24,7 @@ setenv("CMAKE_CXX_COMPILER","CC") setenv("CMAKE_Fortran_COMPILER","ftn") setenv("CMAKE_Platform","gaea.intel") +setenv("CFLAGS","-diag-disable=10441") +setenv("FFLAGS","-diag-disable=10441 -fp-model source") + + diff --git a/modulefiles/srw_common.lua b/modulefiles/srw_common.lua index 782098c62d..41d556374a 100644 --- a/modulefiles/srw_common.lua +++ b/modulefiles/srw_common.lua @@ -1,12 +1,11 @@ -load("jasper/2.0.25") -load("zlib/1.2.11") - +load_any("jasper/2.0.25","jasper/2.0.32") +load_any("zlib/1.2.11","zlib/1.2.13") load("libpng/1.6.37") load_any("netcdf/4.9.2", "netcdf-c/4.9.2") load_any("netcdf/4.9.2", "netcdf-fortran/4.6.0") -load_any("pio/2.5.10", "parallelio/2.5.10") -load_any("esmf/8.4.2", "esmf/8.5.0") +load_any("pio/2.5.10","parallelio/2.5.10") +load("esmf/8.4.2") load("fms/2023.01") load("bufr/11.7.0") @@ -20,13 +19,13 @@ load("w3emc/2.9.2") load_any("gftl-shared/v1.5.0", "gftl-shared/1.5.0") load_any("yafyaml/v0.5.1", "yafyaml/0.5.1") -load_any("mapl/2.35.2-esmf-8.4.2", "mapl/2.35.2-esmf-8.5.0") +load("mapl/2.35.2-esmf-8.4.2") load("nemsio/2.5.4") load("sfcio/1.4.1") load("sigio/2.3.2") load("w3nco/2.4.1") -load("wrf_io/1.2.0") +load_any("wrf_io/1.2.0","wrf-io/1.2.0") load("ncdiag/1.1.1") load("ncio/1.1.2") From 1111dd94bd0802eec0e480322e4359197b20ae0f Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Tue, 5 Sep 2023 12:15:17 -0400 Subject: [PATCH 08/13] update wflow_gaea modulefile --- modulefiles/wflow_gaea.lua | 2 -- 1 file changed, 2 deletions(-) diff --git a/modulefiles/wflow_gaea.lua b/modulefiles/wflow_gaea.lua index 9dd3632a07..2aba14fd82 100644 --- a/modulefiles/wflow_gaea.lua +++ b/modulefiles/wflow_gaea.lua @@ -13,8 +13,6 @@ prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/rocoto/modulefiles") load("rocoto") load("alps") -pushenv("MKLROOT", "/opt/intel/oneapi/mkl/2022.0.2/") ---pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix/20230601") setenv("PMI_NO_PREINITIALIZE","1") if mode() == "load" then From 768ac5ac5ad5f67c1e8b6c374a5ac166c8005c5a Mon Sep 17 00:00:00 2001 From: Natalie Perlin <68030316+natalie-perlin@users.noreply.github.com> Date: Tue, 12 Sep 2023 01:32:40 -0400 Subject: [PATCH 09/13] Update srw_common.lua keep ncdiag/1.1.1 --- modulefiles/srw_common.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/srw_common.lua b/modulefiles/srw_common.lua index 7588a7ad77..41d556374a 100644 --- a/modulefiles/srw_common.lua +++ b/modulefiles/srw_common.lua @@ -27,6 +27,6 @@ load("sigio/2.3.2") load("w3nco/2.4.1") load_any("wrf_io/1.2.0","wrf-io/1.2.0") ---load("ncdiag/1.1.1") +load("ncdiag/1.1.1") load("ncio/1.1.2") load("wgrib2/2.0.8") From 7e409eabd17f959fe0b2195e24030125a52770e6 Mon Sep 17 00:00:00 2001 From: Natalie Perlin <68030316+natalie-perlin@users.noreply.github.com> Date: Tue, 12 Sep 2023 03:55:36 -0400 Subject: [PATCH 10/13] Update miniconda_regional_workflow_cmaq.lua update miniconda3 location in new role-epic space --- modulefiles/tasks/orion/miniconda_regional_workflow_cmaq.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/tasks/orion/miniconda_regional_workflow_cmaq.lua b/modulefiles/tasks/orion/miniconda_regional_workflow_cmaq.lua index 9d84332bd5..890f642185 100644 --- a/modulefiles/tasks/orion/miniconda_regional_workflow_cmaq.lua +++ b/modulefiles/tasks/orion/miniconda_regional_workflow_cmaq.lua @@ -1,4 +1,4 @@ -prepend_path("MODULEPATH","/work/noaa/epic-ps/role-epic-ps/miniconda3/modulefiles") +prepend_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/orion/miniconda3/modulefiles") load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) setenv("SRW_ENV", "regional_workflow_cmaq") From 5c44b068a8b8bdeb1593e49c87e9c1596eed3317 Mon Sep 17 00:00:00 2001 From: Natalie Perlin <68030316+natalie-perlin@users.noreply.github.com> Date: Tue, 12 Sep 2023 03:57:01 -0400 Subject: [PATCH 11/13] Update plot_allvars.local.lua update miniconda3 location in new role-epic space --- modulefiles/tasks/orion/plot_allvars.local.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/tasks/orion/plot_allvars.local.lua b/modulefiles/tasks/orion/plot_allvars.local.lua index 5b7b0afc57..94fed189be 100644 --- a/modulefiles/tasks/orion/plot_allvars.local.lua +++ b/modulefiles/tasks/orion/plot_allvars.local.lua @@ -1,5 +1,5 @@ unload("python") -append_path("MODULEPATH","/work/noaa/epic-ps/role-epic-ps/miniconda3/modulefiles") +append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/orion/miniconda3/modulefiles") load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) setenv("SRW_ENV", "regional_workflow") From 0eef53d6f3b87a5268bb9c44021d53e134bdc838 Mon Sep 17 00:00:00 2001 From: Natalie Perlin <68030316+natalie-perlin@users.noreply.github.com> Date: Tue, 12 Sep 2023 10:42:45 -0400 Subject: [PATCH 12/13] Update wflow_gaea.lua --- modulefiles/wflow_gaea.lua | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modulefiles/wflow_gaea.lua b/modulefiles/wflow_gaea.lua index 2aba14fd82..712e214e78 100644 --- a/modulefiles/wflow_gaea.lua +++ b/modulefiles/wflow_gaea.lua @@ -13,6 +13,13 @@ prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/rocoto/modulefiles") load("rocoto") load("alps") +local MKLROOT="/opt/intel/oneapi/mkl/2023.1.0/" +prepend_path("LD_LIBRARY_PATH",pathJoin(MKLROOT,"lib/intel64")) +pushenv("MKLROOT", MKLROOT) + +pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix/20230601") +pushenv("CRAYPE_LINK_TYPE","dynamic") + setenv("PMI_NO_PREINITIALIZE","1") if mode() == "load" then From d9e2a96089c2a8803bd68c6af4cdce7d5f5d3d0e Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Wed, 13 Sep 2023 14:01:43 -0400 Subject: [PATCH 13/13] gaea modulefiles update, keep configuration for the GSI at the moment --- modulefiles/build_gaea_intel.lua | 7 +++++++ modulefiles/wflow_gaea.lua | 3 --- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/modulefiles/build_gaea_intel.lua b/modulefiles/build_gaea_intel.lua index d8284ce9af..e097516570 100644 --- a/modulefiles/build_gaea_intel.lua +++ b/modulefiles/build_gaea_intel.lua @@ -16,6 +16,13 @@ load("srw_common") -- Need at runtime load("alps") +local MKLROOT="/opt/intel/oneapi/mkl/2023.1.0/" +prepend_path("LD_LIBRARY_PATH",pathJoin(MKLROOT,"lib/intel64")) +pushenv("MKLROOT", MKLROOT) + +pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix/20230601") +pushenv("CRAYPE_LINK_TYPE","dynamic") + setenv("CC","cc") setenv("FC","ftn") setenv("CXX","CC") diff --git a/modulefiles/wflow_gaea.lua b/modulefiles/wflow_gaea.lua index 712e214e78..3e95511f80 100644 --- a/modulefiles/wflow_gaea.lua +++ b/modulefiles/wflow_gaea.lua @@ -16,10 +16,7 @@ load("alps") local MKLROOT="/opt/intel/oneapi/mkl/2023.1.0/" prepend_path("LD_LIBRARY_PATH",pathJoin(MKLROOT,"lib/intel64")) pushenv("MKLROOT", MKLROOT) - pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix/20230601") -pushenv("CRAYPE_LINK_TYPE","dynamic") - setenv("PMI_NO_PREINITIALIZE","1") if mode() == "load" then