From 127d7e47c9e8344dc0692eed870c95cf0b200911 Mon Sep 17 00:00:00 2001 From: GeorgeGayno-NOAA <52789452+GeorgeGayno-NOAA@users.noreply.github.com> Date: Thu, 28 Apr 2022 11:43:44 -0400 Subject: [PATCH 1/2] Download unit test data as part of the CMake build (#630) Update the build system to download all files required for the unit tests. Fixes #619. --- .../workflows/debug-docs-test_coverage.yml | 5 +---- .github/workflows/intel.yml | 5 +---- .github/workflows/linux-mac-nceplibs-mpi.yml | 5 +---- .github/workflows/netcdf-versions.yml | 5 +---- CMakeLists.txt | 4 +--- build_all.sh | 17 +++++++++++++-- cmake/mpiexec.jet | 15 ------------- tests/CMakeLists.txt | 21 +++++++++++++++++++ tests/chgres_cube/CMakeLists.txt | 16 +++++++++++--- tests/chgres_cube/data/files.txt | 6 ------ tests/fvcom_tools/CMakeLists.txt | 16 ++++++++------ tests/fvcom_tools/ftst_readfvcomnetcdf.F90 | 4 ++-- tests/sfc_climo_gen/CMakeLists.txt | 16 ++++++++++++-- tests/sfc_climo_gen/data/files.txt | 4 ---- 14 files changed, 80 insertions(+), 59 deletions(-) delete mode 100755 cmake/mpiexec.jet delete mode 100644 tests/chgres_cube/data/files.txt delete mode 100644 tests/sfc_climo_gen/data/files.txt diff --git a/.github/workflows/debug-docs-test_coverage.yml b/.github/workflows/debug-docs-test_coverage.yml index 6d953cd95..011645c34 100644 --- a/.github/workflows/debug-docs-test_coverage.yml +++ b/.github/workflows/debug-docs-test_coverage.yml @@ -103,10 +103,7 @@ jobs: - name: test run: | - cd ufs_utils/build/tests - wget -i ./chgres_cube/data/files.txt -P ./chgres_cube/data - wget -i ./sfc_climo_gen/data/files.txt -P ./sfc_climo_gen/data - cd ../ + cd ufs_utils/build export LSAN_OPTIONS=suppressions=LSanSuppress.supp ctest --rerun-failed --output-on-failure export PATH="/home/runner/.local/bin:$PATH" diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index 240003849..cbc79472f 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -166,8 +166,5 @@ jobs: - name: test run: | - cd ufs_utils/build/tests - wget -i ./chgres_cube/data/files.txt -P ./chgres_cube/data - wget -i ./sfc_climo_gen/data/files.txt -P ./sfc_climo_gen/data - cd ../ + cd ufs_utils/build ctest --rerun-failed --output-on-failure diff --git a/.github/workflows/linux-mac-nceplibs-mpi.yml b/.github/workflows/linux-mac-nceplibs-mpi.yml index 68d9da6ae..205e41a34 100644 --- a/.github/workflows/linux-mac-nceplibs-mpi.yml +++ b/.github/workflows/linux-mac-nceplibs-mpi.yml @@ -221,10 +221,7 @@ jobs: - name: test run: | - cd ufs_utils/build/tests - wget -i ./chgres_cube/data/files.txt -P ./chgres_cube/data - wget -i ./sfc_climo_gen/data/files.txt -P ./sfc_climo_gen/data - cd ../ + cd ufs_utils/build # Oversubscribe for OpenMPI to run more processes than CPUs export OMPI_MCA_rmaps_base_oversubscribe=1 ctest --rerun-failed --output-on-failure diff --git a/.github/workflows/netcdf-versions.yml b/.github/workflows/netcdf-versions.yml index 98ecd746a..42eb28417 100644 --- a/.github/workflows/netcdf-versions.yml +++ b/.github/workflows/netcdf-versions.yml @@ -154,9 +154,6 @@ jobs: - name: test run: | - cd ufs_utils/build/tests - wget -i ./chgres_cube/data/files.txt -P ./chgres_cube/data - wget -i ./sfc_climo_gen/data/files.txt -P ./sfc_climo_gen/data - cd ../ + cd ufs_utils/build export LSAN_OPTIONS=suppressions=LSanSuppress.supp ctest --rerun-failed --output-on-failure diff --git a/CMakeLists.txt b/CMakeLists.txt index 68869d5c2..f57f1c9eb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,8 +23,6 @@ if(NOT CMAKE_BUILD_TYPE MATCHES "^(Debug|Release|RelWithDebInfo|MinSizeRel)$") CACHE STRING "Choose the type of build." FORCE) set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") - message(STATUS "Set BUILD_TESTING to YES and build unit testing package under tests") - set(BUILD_TESTING "YES") endif() # Set compiler flags. @@ -92,6 +90,6 @@ add_subdirectory(sorc) # Run unit tests. include(CTest) if(BUILD_TESTING) + message(STATUS "Set BUILD_TESTING to YES and build unit testing package under tests") add_subdirectory(tests) endif() - diff --git a/build_all.sh b/build_all.sh index fe457c9ca..0a8de2d7c 100755 --- a/build_all.sh +++ b/build_all.sh @@ -26,8 +26,18 @@ else set -x fi -CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=../ -DEMC_EXEC_DIR=ON" -#CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=../ -DEMC_EXEC_DIR=ON -DENABLE_DOCS=ON" +# The unit test data download is part of the build system. Not all machines can +# access the EMC ftp site, so turn off the build (-DBUILD_TESTING=OFF) of the units tests accordingly. +# Those with access to the EMC ftp site are: Orion, Hera, WCOSS-Cray, WCOSS-Dell. + +if [[ "$target" == "hera" || "$target" == "orion" || "$target" == "wcoss_cray" || "$target" == "wcoss_dell_p3" ]]; then + CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=../ -DEMC_EXEC_DIR=ON -DBUILD_TESTING=OFF" + #CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=../ -DEMC_EXEC_DIR=ON -DBUILD_TESTING=ON" + #CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=../ -DEMC_EXEC_DIR=ON -DENABLE_DOCS=ON -DBUILD_TESTING=ON" +else + CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=../ -DEMC_EXEC_DIR=ON -DBUILD_TESTING=OFF" + #CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=../ -DEMC_EXEC_DIR=ON -DENABLE_DOCS=ON -DBUILD_TESTING=OFF" +fi rm -fr ./build mkdir ./build && cd ./build @@ -37,4 +47,7 @@ cmake .. ${CMAKE_FLAGS} make -j 8 VERBOSE=1 make install +#make test +#ctest -I 4,5 + exit diff --git a/cmake/mpiexec.jet b/cmake/mpiexec.jet deleted file mode 100755 index 332b33e29..000000000 --- a/cmake/mpiexec.jet +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# -# Arguments: -# -# $1 - Number of MPI Tasks -# $2+ - Executable and its arguments -# - -ACCOUNT= -QOS=debug - -NP=$1 -shift - -srun -A $ACCOUNT -q $QOS -n $NP $@ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 894b01b5f..67f3e3e0b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -3,7 +3,28 @@ # # Ed Hartnett 2/11/21 +# This function is used to download unit test data. +# It takes two arguments, the URL and the file to +# be downloaded. + +function(PULL_DATA THE_URL THE_FILE) + if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/data/${THE_FILE}") + file(DOWNLOAD + ${THE_URL}/${THE_FILE} + ${CMAKE_CURRENT_BINARY_DIR}/data/${THE_FILE} + SHOW_PROGRESS + STATUS status + INACTIVITY_TIMEOUT 30 + ) + list(GET status 0 status_num) + if(NOT status_num EQUAL 0 OR NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/data/${THE_FILE}") + message(FATAL_ERROR "Could not download ${THE_FILE}") + endif() + endif() +endfunction() + # Add the test subdirecotries. +# fvcom test only works for Intel. Comment out for now. #add_subdirectory(fvcom_tools) add_subdirectory(filter_topo) add_subdirectory(chgres_cube) diff --git a/tests/chgres_cube/CMakeLists.txt b/tests/chgres_cube/CMakeLists.txt index 73274ca76..d6d46f95a 100644 --- a/tests/chgres_cube/CMakeLists.txt +++ b/tests/chgres_cube/CMakeLists.txt @@ -3,7 +3,19 @@ # # George Gayno, Lin Gan, Ed Hartnett, Larissa Reames -# Include cmake to allow parallel I/O tests. +set(CHGRES_URL "https://ftp.emc.ncep.noaa.gov/static_files/public/UFS/ufs_utils/unit_tests/chgres_cube") + +set(V16SFC_FILE "gfs.v16.sfc.history.nc") +set(V16SFC_FILE2 "gfs.v16.sfc2.history.nc") +set(V16ATM_FILE "gfs.v16.atm.history.nc") +set(V14SFC_FILE "gfs.v14.sfc.history.nemsio") +set(V15SFC_FILE "gfs.v15.sfc.history.nemsio") +set(GFS_GRIB_FILE "gfs.t00z.pgrb2.0p50.f000") + +foreach(THE_FILE IN LISTS V16SFC_FILE V16SFC_FILE2 V16ATM_FILE V14SFC_FILE V15SFC_FILE GFS_GRIB_FILE) + PULL_DATA(${CHGRES_URL} ${THE_FILE}) +endforeach() + include (LibMPI) if(CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel)$") @@ -16,8 +28,6 @@ include_directories(${PROJECT_SOURCE_DIR}) # Copy necessary test files from the source data directory to the # build data directory. -execute_process( COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_SOURCE_DIR}/data/files.txt ${CMAKE_CURRENT_BINARY_DIR}/data/files.txt) execute_process( COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/data/config_fv3_tiled.nml ${CMAKE_CURRENT_BINARY_DIR}/data/config_fv3_tiled.nml) execute_process( COMMAND ${CMAKE_COMMAND} -E copy diff --git a/tests/chgres_cube/data/files.txt b/tests/chgres_cube/data/files.txt deleted file mode 100644 index e214957a4..000000000 --- a/tests/chgres_cube/data/files.txt +++ /dev/null @@ -1,6 +0,0 @@ -https://ftp.emc.ncep.noaa.gov/static_files/public/UFS/ufs_utils/unit_tests/chgres_cube/gfs.v16.sfc.history.nc -https://ftp.emc.ncep.noaa.gov/static_files/public/UFS/ufs_utils/unit_tests/chgres_cube/gfs.v16.sfc2.history.nc -https://ftp.emc.ncep.noaa.gov/static_files/public/UFS/ufs_utils/unit_tests/chgres_cube/gfs.v16.atm.history.nc -https://ftp.emc.ncep.noaa.gov/static_files/public/UFS/ufs_utils/unit_tests/chgres_cube/gfs.v14.sfc.history.nemsio -https://ftp.emc.ncep.noaa.gov/static_files/public/UFS/ufs_utils/unit_tests/chgres_cube/gfs.v15.sfc.history.nemsio -https://ftp.emc.ncep.noaa.gov/static_files/public/UFS/ufs_utils/unit_tests/chgres_cube/gfs.t00z.pgrb2.0p50.f000 diff --git a/tests/fvcom_tools/CMakeLists.txt b/tests/fvcom_tools/CMakeLists.txt index 50ac42441..125d2c956 100644 --- a/tests/fvcom_tools/CMakeLists.txt +++ b/tests/fvcom_tools/CMakeLists.txt @@ -1,8 +1,17 @@ -# This is the cmake build file for the tests directory of the +# This is the cmake build file for the fvcom_tools test directory of the # UFS_UTILS project. # # David Wright +set(FVCOM_URL "https://ftp.emc.ncep.noaa.gov/static_files/public/UFS/ufs_utils/unit_tests/fvcom_tools") + +set(FVCOM_FILE "fvcom_unittest.nc") +set(SFCDATA_FILE "sfcdata_unittest.nc") + +foreach(THE_FILE IN LISTS FVCOM_FILE SFCDATA_FILE) + PULL_DATA(${FVCOM_URL} ${THE_FILE}) +endforeach() + if(CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel)$") set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -r8 -assume byterecl") elseif(CMAKE_Fortran_COMPILER_ID MATCHES "^(GNU)$") @@ -13,14 +22,9 @@ include_directories(${PROJECT_SOURCE_DIR}) # Copy necessary test files from the source data directory to the # build data directory. -execute_process( COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_SOURCE_DIR}/data/sfcdata_unittest.nc ${CMAKE_CURRENT_BINARY_DIR}/sfcdata_unittest.nc) -execute_process( COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_SOURCE_DIR}/data/fvcom_unittest.nc ${CMAKE_CURRENT_BINARY_DIR}/fvcom_unittest.nc) execute_process( COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/LSanSuppress.supp ${CMAKE_CURRENT_BINARY_DIR}/LSanSuppress.supp) add_executable(ftst_readfvcomnetcdf ftst_readfvcomnetcdf.F90) add_test(NAME fvcom_tools-ftst_readfvcomnetcdf COMMAND ftst_readfvcomnetcdf) target_link_libraries(ftst_readfvcomnetcdf fvcom_tools_lib) - diff --git a/tests/fvcom_tools/ftst_readfvcomnetcdf.F90 b/tests/fvcom_tools/ftst_readfvcomnetcdf.F90 index d43fb1f7f..a472f660f 100644 --- a/tests/fvcom_tools/ftst_readfvcomnetcdf.F90 +++ b/tests/fvcom_tools/ftst_readfvcomnetcdf.F90 @@ -74,8 +74,8 @@ program readfvcomnetcdf print*,"Starting test of fvcom_tools." !Set default file names, cold start, and time str - fv3file = 'sfcdata_unittest.nc' - fvcomfile = 'fvcom_unittest.nc' + fv3file = './data/sfcdata_unittest.nc' + fvcomfile = './data/fvcom_unittest.nc' wcstart = 'cold' inputFVCOMselStr = '3333-44-55T66:77:88.000000' t1 = 1 diff --git a/tests/sfc_climo_gen/CMakeLists.txt b/tests/sfc_climo_gen/CMakeLists.txt index 734dd1f4d..82e90207f 100644 --- a/tests/sfc_climo_gen/CMakeLists.txt +++ b/tests/sfc_climo_gen/CMakeLists.txt @@ -3,6 +3,20 @@ # # George Gayno, Lin Gan, Ed Hartnett, Larissa Reames +set(SFCGEN_URL "https://ftp.emc.ncep.noaa.gov/static_files/public/UFS/ufs_utils/unit_tests/sfc_climo_gen") +set(MOSAIC_FILE "C450_mosaic.nc") +set(GRID_FILE "C450_grid.tile7.nc") +set(ORO_FILE "C450_oro_data.tile7.nc") + +foreach(THE_FILE IN LISTS MOSAIC_FILE GRID_FILE ORO_FILE) + PULL_DATA(${SFCGEN_URL} ${THE_FILE}) +endforeach() + +set(SFCGEN2_URL "https://ftp.emc.ncep.noaa.gov/static_files/public/UFS/GFS/fix/fix_sfc_climo") +set(VEG_FILE "vegetation_type.viirs.igbp.0.1.nc") + +PULL_DATA(${SFCGEN2_URL} ${VEG_FILE}) + # Include cmake to allow parallel I/O tests. include (LibMPI) @@ -16,8 +30,6 @@ include_directories(${PROJECT_SOURCE_DIR}) # Copy necessary test files from the source data directory to the # build data directory. -execute_process( COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_SOURCE_DIR}/data/files.txt ${CMAKE_CURRENT_BINARY_DIR}/data/files.txt) execute_process( COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/LSanSuppress.supp ${CMAKE_CURRENT_BINARY_DIR}/LSanSuppress.supp) diff --git a/tests/sfc_climo_gen/data/files.txt b/tests/sfc_climo_gen/data/files.txt deleted file mode 100644 index c5b8bd439..000000000 --- a/tests/sfc_climo_gen/data/files.txt +++ /dev/null @@ -1,4 +0,0 @@ -https://ftp.emc.ncep.noaa.gov/static_files/public/UFS/ufs_utils/unit_tests/sfc_climo_gen/C450_mosaic.nc -https://ftp.emc.ncep.noaa.gov/static_files/public/UFS/ufs_utils/unit_tests/sfc_climo_gen/C450_grid.tile7.nc -https://ftp.emc.ncep.noaa.gov/static_files/public/UFS/ufs_utils/unit_tests/sfc_climo_gen/C450_oro_data.tile7.nc -https://ftp.emc.ncep.noaa.gov/static_files/public/UFS/GFS/fix/fix_sfc_climo/vegetation_type.viirs.igbp.0.1.nc From b23a25a218deb20113b6cf18002525df2e06841d Mon Sep 17 00:00:00 2001 From: GeorgeGayno-NOAA <52789452+GeorgeGayno-NOAA@users.noreply.github.com> Date: Tue, 3 May 2022 14:56:15 -0400 Subject: [PATCH 2/2] Host doxygen documentation for multiple releases (#644) Update the user_guide.md file to list a table of contents for each release. Fixes #610. --- docs/user_guide.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/user_guide.md b/docs/user_guide.md index dc70b4466..624c2f66d 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -8,6 +8,13 @@ Utilities for the NCEP models. This is part of the The UFS_UTILS code can be found here: https://github.com/ufs-community/UFS_UTILS. +## Documentation for Previous Versions of UFS_UTILS + +* [UFS_UTILS Version 1.6.0](ver-1.6.0/index.html) +* [UFS_UTILS Version 1.5.0](ver-1.5.0/index.html) +* [UFS_UTILS Version 1.4.0](ver-1.4.0/index.html) +* [UFS_UTILS Version 1.3.0](ver-1.3.0/index.html) + ## The Utilities - chgres_cube - Creates cold