From ea853bd921eddfa362b6df69cf1a3174c164e964 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Tue, 3 Oct 2023 18:40:41 -0500 Subject: [PATCH 1/4] Replace CDAT --- setup.py | 7 ++++++- zppy/__main__.py | 16 ++++++++++++++++ zppy/templates/e3sm_diags.bash | 4 ++-- zppy/templates/global_time_series.bash | 6 +++--- zppy/templates/readTS.py | 7 +++---- 5 files changed, 30 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index 3f5c80c0..1798f077 100644 --- a/setup.py +++ b/setup.py @@ -30,5 +30,10 @@ def package_files(directory, prefixes, extensions): python_requires=">=3.6", packages=find_packages(include=["zppy", "zppy.*"]), package_data={"": data_files}, - entry_points={"console_scripts": ["zppy=zppy.__main__:main"]}, + entry_points={ + "console_scripts": [ + "zppy=zppy.__main__:main", + "zppy_cdscan_replacement=zppy.__main__:zppy_cdscan_replacement", + ] + }, ) diff --git a/zppy/__main__.py b/zppy/__main__.py index adf8d067..1c13fcd8 100644 --- a/zppy/__main__.py +++ b/zppy/__main__.py @@ -3,8 +3,10 @@ import importlib import io import os +import sys from typing import List +import pandas from configobj import ConfigObj from mache import MachineInfo from validate import Validator @@ -246,3 +248,17 @@ def _validate_config(config): ) else: print("Configuration file validation passed.") + + +def zppy_cdscan_replacement(): + output_file = sys.argv[1] + input_files = sys.argv[2:] + if len(input_files) == 1: + files = [] + # Convert text file of files to a list of files. + with open(input_files[0]) as f: + for line in f: + files.append(line.strip()) + input_files = files + dataframe = pandas.read_table(input_files) + dataframe.to_xml(output_file) diff --git a/zppy/templates/e3sm_diags.bash b/zppy/templates/e3sm_diags.bash index 4823e0ff..7646e3eb 100644 --- a/zppy/templates/e3sm_diags.bash +++ b/zppy/templates/e3sm_diags.bash @@ -108,7 +108,7 @@ create_links_ts() # xml file will cover the whole period from year1 to year2 xml_name=${v}_${begin_year}01_${end_year}12.xml export CDMS_NO_MPI=true - cdscan -x ${xml_name} -f ${v}_files.txt + zppy_cdscan_replacement ${xml_name} ${v}_files.txt if [ $? != 0 ]; then cd {{ scriptDir }} echo "ERROR (${error_num})" > {{ prefix }}.status @@ -129,7 +129,7 @@ create_links_ts_rof() cd ${ts_rof_dir_destination} v="RIVER_DISCHARGE_OVER_LAND_LIQ" xml_name=${v}_${begin_year}01_${end_year}12.xml - cdscan -x ${xml_name} ${ts_rof_dir_source}/${v}_*.nc + zppy_cdscan_replacement ${xml_name} ${ts_rof_dir_source}/${v}_*.nc if [ $? != 0 ]; then cd {{ scriptDir }} echo "ERROR (${error_num})" > {{ prefix }}.status diff --git a/zppy/templates/global_time_series.bash b/zppy/templates/global_time_series.bash index 2154e66b..52aa76d2 100644 --- a/zppy/templates/global_time_series.bash +++ b/zppy/templates/global_time_series.bash @@ -42,7 +42,7 @@ use_atm={{ use_atm }} if [[ ${use_atm,,} == "true" ]]; then echo 'Create xml files for atm' cd ${case_dir}/post/atm/glb/ts/monthly/${ts_num_years}yr - cdscan -x glb.xml *.nc + zppy_cdscan_replacement glb.xml *.nc if [ $? != 0 ]; then cd {{ scriptDir }} echo 'ERROR (1)' > {{ prefix }}.status @@ -54,7 +54,7 @@ use_lnd={{ use_lnd }} if [[ ${use_lnd,,} == "true" ]]; then echo 'Create xml files for lnd' cd ${case_dir}/post/lnd/glb/ts/monthly/${ts_num_years}yr - cdscan -x glb.xml *.nc + zppy_cdscan_replacement -x glb.xml *.nc if [ $? != 0 ]; then cd {{ scriptDir }} echo 'ERROR (2)' > {{ prefix }}.status @@ -78,7 +78,7 @@ if [[ ${use_ocn,,} == "true" ]]; then echo 'Create xml for for ocn' export CDMS_NO_MPI=true cd ${case_dir}/post/ocn/glb/ts/monthly/${ts_num_years}yr - cdscan -x glb.xml mpaso.glb*.nc + zppy_cdscan_replacement glb.xml mpaso.glb*.nc if [ $? != 0 ]; then cd {{ scriptDir }} echo 'ERROR (4)' > {{ prefix }}.status diff --git a/zppy/templates/readTS.py b/zppy/templates/readTS.py index 58f4c712..e0e4374b 100644 --- a/zppy/templates/readTS.py +++ b/zppy/templates/readTS.py @@ -1,5 +1,4 @@ -import cdms2 -import cdutil +import xcdat class TS(object): @@ -7,7 +6,7 @@ def __init__(self, filename): self.filename = filename - self.f = cdms2.open(filename) + self.f = xcdat.open_dataset(filename) def __del__(self): @@ -66,6 +65,6 @@ def globalAnnual(self, var): units = v.units # Annual average - v = cdutil.YEAR(v) + v = self.f.temporal.group_average(v, "year") return v, units From 05e432c34e76d56a201887d5ea11387f567ecfd9 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Tue, 14 May 2024 17:55:06 -0500 Subject: [PATCH 2/4] types without xml --- setup.py | 7 +-- .../generated/test_bundles_chrysalis.cfg | 4 +- .../generated/test_complete_run_chrysalis.cfg | 8 +-- .../generated/test_debug_chrysalis.cfg | 53 +++++++++++-------- ...update_bundles_expected_files_chrysalis.sh | 4 +- ...e_complete_run_expected_files_chrysalis.sh | 2 +- tests/integration/template_debug.cfg | 51 ++++++++++-------- tests/integration/utils.py | 2 +- zppy/__main__.py | 16 ------ zppy/templates/coupled_global.py | 20 ++----- zppy/templates/e3sm_diags.bash | 4 +- zppy/templates/global_time_series.bash | 31 ----------- zppy/templates/readTS.py | 29 +++++++--- 13 files changed, 98 insertions(+), 133 deletions(-) diff --git a/setup.py b/setup.py index 1798f077..3f5c80c0 100644 --- a/setup.py +++ b/setup.py @@ -30,10 +30,5 @@ def package_files(directory, prefixes, extensions): python_requires=">=3.6", packages=find_packages(include=["zppy", "zppy.*"]), package_data={"": data_files}, - entry_points={ - "console_scripts": [ - "zppy=zppy.__main__:main", - "zppy_cdscan_replacement=zppy.__main__:zppy_cdscan_replacement", - ] - }, + entry_points={"console_scripts": ["zppy=zppy.__main__:main"]}, ) diff --git a/tests/integration/generated/test_bundles_chrysalis.cfg b/tests/integration/generated/test_bundles_chrysalis.cfg index bd465e48..15ef3271 100644 --- a/tests/integration/generated/test_bundles_chrysalis.cfg +++ b/tests/integration/generated/test_bundles_chrysalis.cfg @@ -7,11 +7,11 @@ input = "/lcrc/group/e3sm/ac.forsyth2/E3SMv2/v2.LR.historical_0201" input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_bundles.py -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/unique_id/v2.LR.historical_0201" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/test-346/v2.LR.historical_0201" partition = "compute" qos = "regular" walltime = "07:00:00" -www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/unique_id" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/test-346" [bundle] diff --git a/tests/integration/generated/test_complete_run_chrysalis.cfg b/tests/integration/generated/test_complete_run_chrysalis.cfg index 358baa50..d3559f22 100644 --- a/tests/integration/generated/test_complete_run_chrysalis.cfg +++ b/tests/integration/generated/test_complete_run_chrysalis.cfg @@ -7,10 +7,10 @@ input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201" input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/test-346/v2.LR.historical_0201" partition = "debug" qos = "regular" -www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/unique_id" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/test-346" [climo] active = True @@ -134,7 +134,7 @@ years = "1850:1854:2", "1850:1854:4", ref_name = "v2.LR.historical_0201" ref_start_yr = 1850 ref_years = "1850-1851", - reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201/post/atm/180x360_aave/clim" + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/test-346/v2.LR.historical_0201/post/atm/180x360_aave/clim" run_type = "model_vs_model" short_ref_name = "v2.LR.historical_0201" swap_test_ref = False @@ -155,7 +155,7 @@ years = "1850:1854:2", "1850:1854:4", ref_name = "v2.LR.historical_0201" ref_start_yr = 1850 ref_years = "1850-1851", - reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201/post/lnd/180x360_aave/clim" + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/test-346/v2.LR.historical_0201/post/lnd/180x360_aave/clim" run_type = "model_vs_model" sets = "lat_lon_land", short_ref_name = "same simulation" diff --git a/tests/integration/generated/test_debug_chrysalis.cfg b/tests/integration/generated/test_debug_chrysalis.cfg index 3e3ff988..30de023d 100644 --- a/tests/integration/generated/test_debug_chrysalis.cfg +++ b/tests/integration/generated/test_debug_chrysalis.cfg @@ -12,46 +12,53 @@ environment_commands = "" input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201" input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" -# To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/unique_id/v2.LR.historical_0201" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/test-346/v2.LR.historical_0201" partition = "debug" qos = "regular" -www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_debug_www/unique_id" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_debug_www/test-346" [ts] active = True +e3sm_to_cmip_environment_commands = "" walltime = "00:30:00" years = "1850:1854:2", - [[ atm_monthly_180x360_aave ]] + [[ atm_monthly_glb ]] + # Note global average won't work for 3D variables. frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" - ts_fmt = "cmip" + mapping_file = "glb" + years = "1850:1860:5", - [[ atm_daily_180x360_aave ]] - frequency = "daily" - input_files = "eam.h1" - input_subdir = "archive/atm/hist" - vars = "PRECT" - - [[ land_monthly ]] - e3sm_to_cmip_environment_commands = "" - extra_vars = "landfrac" + [[ lnd_monthly_glb ]] frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" + mapping_file = "glb" vars = "LAISHA,LAISUN" - #vars = "FSH,RH2M" - ts_fmt = "cmip" + years = "1850:1860:5", -[ilamb] +[mpas_analysis] active = True -grid = '180x360_aave' -nodes = 8 +anomalyRefYear = 1850 +climo_years ="1850-1854", "1855-1860", +enso_years = "1850-1854", "1855-1860", +mesh = "EC30to60E2r2" +parallelTaskCount = 6 partition = "compute" -short_name = 'v2.LR.historical_0201' -ts_num_years = 2 -years = "1850:1854:2", +qos = "regular" +ts_years = "1850-1854", "1850-1860", +walltime = "00:30:00" - [[ land_monthly ]] +[global_time_series] +active = True +climo_years ="1850-1854", "1855-1860", +experiment_name = "v2.LR.historical_0201" +figstr = "v2_historical_0201" +moc_file=mocTimeSeries_1850-1860.nc +plots_lnd = "LAISHA,LAISUN" +ts_num_years = 5 +ts_years = "1850-1854", "1850-1860", +walltime = "00:30:00" +years = "1850-1860", diff --git a/tests/integration/generated/update_bundles_expected_files_chrysalis.sh b/tests/integration/generated/update_bundles_expected_files_chrysalis.sh index ae3b50cb..f4c4d6e4 100755 --- a/tests/integration/generated/update_bundles_expected_files_chrysalis.sh +++ b/tests/integration/generated/update_bundles_expected_files_chrysalis.sh @@ -6,9 +6,9 @@ rm -rf /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles # Your output will now become the new expectation. # Copy output so you don't have to rerun zppy to generate the output. -cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/unique_id/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles +cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/test-346/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles mkdir -p /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files -cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/unique_id/v2.LR.historical_0201/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files +cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/test-346/v2.LR.historical_0201/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files zppy_top_level=$(pwd) cd /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles # Remove the image check failures, so they don't end up in the expected files. diff --git a/tests/integration/generated/update_complete_run_expected_files_chrysalis.sh b/tests/integration/generated/update_complete_run_expected_files_chrysalis.sh index 0ed4a48a..9c63989d 100755 --- a/tests/integration/generated/update_complete_run_expected_files_chrysalis.sh +++ b/tests/integration/generated/update_complete_run_expected_files_chrysalis.sh @@ -6,7 +6,7 @@ rm -rf /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run # Your output will now become the new expectation. # Copy output so you don't have to rerun zppy to generate the output. -cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/unique_id/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run +cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/test-346/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run zppy_top_level=$(pwd) cd /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run # Remove the image check failures, so they don't end up in the expected files. diff --git a/tests/integration/template_debug.cfg b/tests/integration/template_debug.cfg index d9a64bf5..baded2b0 100644 --- a/tests/integration/template_debug.cfg +++ b/tests/integration/template_debug.cfg @@ -12,46 +12,53 @@ environment_commands = "#expand environment_commands#" input = "#expand user_input#/E3SMv2/v2.LR.historical_0201" input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" -# To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py output = "#expand user_output#zppy_test_debug_output/#expand unique_id#/v2.LR.historical_0201" partition = "#expand partition_short#" qos = "#expand qos_short#" -www = "#expand user_www#zppy_debug_www/#expand unique_id#" +www = "#expand user_www#zppy_test_debug_www/#expand unique_id#" [ts] active = True +e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" walltime = "00:30:00" years = "1850:1854:2", - [[ atm_monthly_180x360_aave ]] + [[ atm_monthly_glb ]] + # Note global average won't work for 3D variables. frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" - ts_fmt = "cmip" + mapping_file = "glb" + years = "1850:1860:5", - [[ atm_daily_180x360_aave ]] - frequency = "daily" - input_files = "eam.h1" - input_subdir = "archive/atm/hist" - vars = "PRECT" - - [[ land_monthly ]] - e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" - extra_vars = "landfrac" + [[ lnd_monthly_glb ]] frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" + mapping_file = "glb" vars = "LAISHA,LAISUN" - #vars = "FSH,RH2M" - ts_fmt = "cmip" + years = "1850:1860:5", -[ilamb] +[mpas_analysis] active = True -grid = '180x360_aave' -nodes = 8 +anomalyRefYear = 1850 +climo_years ="1850-1854", "1855-1860", +enso_years = "1850-1854", "1855-1860", +mesh = "EC30to60E2r2" +parallelTaskCount = 6 partition = "#expand partition_long#" -short_name = 'v2.LR.historical_0201' -ts_num_years = 2 -years = "1850:1854:2", +qos = "#expand qos_long#" +ts_years = "1850-1854", "1850-1860", +walltime = "#expand mpas_analysis_walltime#" - [[ land_monthly ]] +[global_time_series] +active = True +climo_years ="1850-1854", "1855-1860", +experiment_name = "v2.LR.historical_0201" +figstr = "v2_historical_0201" +moc_file=mocTimeSeries_1850-1860.nc +plots_lnd = "LAISHA,LAISUN" +ts_num_years = 5 +ts_years = "1850-1854", "1850-1860", +walltime = "00:30:00" +years = "1850-1860", diff --git a/tests/integration/utils.py b/tests/integration/utils.py index b8f1e74c..9f4282fc 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -7,7 +7,7 @@ from mache import MachineInfo from PIL import Image, ImageChops, ImageDraw -UNIQUE_ID = "unique_id" +UNIQUE_ID = "test-346" # Image checking ########################################################## diff --git a/zppy/__main__.py b/zppy/__main__.py index 1c13fcd8..adf8d067 100644 --- a/zppy/__main__.py +++ b/zppy/__main__.py @@ -3,10 +3,8 @@ import importlib import io import os -import sys from typing import List -import pandas from configobj import ConfigObj from mache import MachineInfo from validate import Validator @@ -248,17 +246,3 @@ def _validate_config(config): ) else: print("Configuration file validation passed.") - - -def zppy_cdscan_replacement(): - output_file = sys.argv[1] - input_files = sys.argv[2:] - if len(input_files) == 1: - files = [] - # Convert text file of files to a list of files. - with open(input_files[0]) as f: - for line in f: - files.append(line.strip()) - input_files = files - dataframe = pandas.read_table(input_files) - dataframe.to_xml(output_file) diff --git a/zppy/templates/coupled_global.py b/zppy/templates/coupled_global.py index 4957ba55..e21dfa48 100644 --- a/zppy/templates/coupled_global.py +++ b/zppy/templates/coupled_global.py @@ -740,32 +740,22 @@ def run(parameters, rgn): # noqa: C901 { "atmos": None if not use_atmos - else "{}/post/atm/glb/ts/monthly/{}yr/glb.xml".format( - case_dir, ts_num_years - ), + else "{}/post/atm/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years), "ice": None if not plots_ice - else "{}/post/ice/glb/ts/monthly/{}yr/glb.xml".format( - case_dir, ts_num_years - ), + else "{}/post/ice/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years), "land": None if not plots_lnd - else "{}/post/lnd/glb/ts/monthly/{}yr/glb.xml".format( - case_dir, ts_num_years - ), + else "{}/post/lnd/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years), "ocean": None if not use_ocn - else "{}/post/ocn/glb/ts/monthly/{}yr/glb.xml".format( - case_dir, ts_num_years - ), + else "{}/post/ocn/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years), "moc": None if not use_ocn else "{}/post/ocn/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years), "vol": None if not use_ocn - else "{}/post/ocn/glb/ts/monthly/{}yr/glb.xml".format( - case_dir, ts_num_years - ), + else "{}/post/ocn/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years), "name": experiment_name, "yoffset": 0.0, "yr": ([year1, year2],), diff --git a/zppy/templates/e3sm_diags.bash b/zppy/templates/e3sm_diags.bash index 7646e3eb..4f70005b 100644 --- a/zppy/templates/e3sm_diags.bash +++ b/zppy/templates/e3sm_diags.bash @@ -108,7 +108,7 @@ create_links_ts() # xml file will cover the whole period from year1 to year2 xml_name=${v}_${begin_year}01_${end_year}12.xml export CDMS_NO_MPI=true - zppy_cdscan_replacement ${xml_name} ${v}_files.txt + cdscan ${xml_name} ${v}_files.txt if [ $? != 0 ]; then cd {{ scriptDir }} echo "ERROR (${error_num})" > {{ prefix }}.status @@ -129,7 +129,7 @@ create_links_ts_rof() cd ${ts_rof_dir_destination} v="RIVER_DISCHARGE_OVER_LAND_LIQ" xml_name=${v}_${begin_year}01_${end_year}12.xml - zppy_cdscan_replacement ${xml_name} ${ts_rof_dir_source}/${v}_*.nc + cdscan ${xml_name} ${ts_rof_dir_source}/${v}_*.nc if [ $? != 0 ]; then cd {{ scriptDir }} echo "ERROR (${error_num})" > {{ prefix }}.status diff --git a/zppy/templates/global_time_series.bash b/zppy/templates/global_time_series.bash index 52aa76d2..22dee4d7 100644 --- a/zppy/templates/global_time_series.bash +++ b/zppy/templates/global_time_series.bash @@ -39,28 +39,7 @@ global_ts_dir={{ global_time_series_dir }} export CDMS_NO_MPI=true use_atm={{ use_atm }} -if [[ ${use_atm,,} == "true" ]]; then - echo 'Create xml files for atm' - cd ${case_dir}/post/atm/glb/ts/monthly/${ts_num_years}yr - zppy_cdscan_replacement glb.xml *.nc - if [ $? != 0 ]; then - cd {{ scriptDir }} - echo 'ERROR (1)' > {{ prefix }}.status - exit 1 - fi -fi - use_lnd={{ use_lnd }} -if [[ ${use_lnd,,} == "true" ]]; then - echo 'Create xml files for lnd' - cd ${case_dir}/post/lnd/glb/ts/monthly/${ts_num_years}yr - zppy_cdscan_replacement -x glb.xml *.nc - if [ $? != 0 ]; then - cd {{ scriptDir }} - echo 'ERROR (2)' > {{ prefix }}.status - exit 2 - fi -fi use_ocn={{ use_ocn }} if [[ ${use_ocn,,} == "true" ]]; then @@ -75,16 +54,6 @@ if [[ ${use_ocn,,} == "true" ]]; then exit 3 fi - echo 'Create xml for for ocn' - export CDMS_NO_MPI=true - cd ${case_dir}/post/ocn/glb/ts/monthly/${ts_num_years}yr - zppy_cdscan_replacement glb.xml mpaso.glb*.nc - if [ $? != 0 ]; then - cd {{ scriptDir }} - echo 'ERROR (4)' > {{ prefix }}.status - exit 4 - fi - echo 'Copy moc file' cd ${case_dir}/post/analysis/mpas_analysis/cache/timeseries/moc cp ${moc_file} ../../../../../ocn/glb/ts/monthly/${ts_num_years}yr/ diff --git a/zppy/templates/readTS.py b/zppy/templates/readTS.py index e0e4374b..a66db0e8 100644 --- a/zppy/templates/readTS.py +++ b/zppy/templates/readTS.py @@ -1,25 +1,33 @@ -import xcdat +from typing import Optional, Tuple + +import xarray class TS(object): - def __init__(self, filename): + def __init__(self, directory): - self.filename = filename + self.directory: str = directory - self.f = xcdat.open_dataset(filename) + # directory will be of the form `{case_dir}/post//glb/ts/monthly/{ts_num_years}yr` + self.f: xarray.core.dataset.Dataset = xarray.open_mfdataset(f"{directory}/*.nc") + # Refactor note: `self.f = cdms2.open(filename)` gave `cdms2.dataset.Dataset` def __del__(self): self.f.close() - def globalAnnual(self, var): + def globalAnnual( + self, var: str + ) -> Tuple[xarray.core.dataarray.DataArray, Optional[str]]: - units = None + units: Optional[str] = None # Constants, from AMWG diagnostics Lv = 2.501e6 Lf = 3.337e5 + v: xarray.core.dataarray.DataArray + # Is this a derived variable? if var == "RESTOM": @@ -61,10 +69,15 @@ def globalAnnual(self, var): # Non-derived variables # Read variable - v = self.f(var) + v = self.f.data_vars[var] + # Refactor note: `v = self.f(var)` gave `cdms2.tvariable.TransientVariable` units = v.units # Annual average - v = self.f.temporal.group_average(v, "year") + + # Refactor note: `AttributeError: 'Dataset' object has no attribute 'temporal'` seems to always occur + # Regardless if using CDAT or not, if using as object or class method. + # v = self.f.temporal.group_average(v, "year") + v = xarray.Dataset.temporal.group_average(v, "year") return v, units From 0c233d7f30432a89913038cc01bf893f81755a31 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 15 May 2024 19:09:10 -0500 Subject: [PATCH 3/4] glb ts without cdat --- zppy/templates/coupled_global.py | 51 ++++++++++++++++++++++++++------ zppy/templates/readTS.py | 22 +++++--------- 2 files changed, 50 insertions(+), 23 deletions(-) diff --git a/zppy/templates/coupled_global.py b/zppy/templates/coupled_global.py index e21dfa48..f338eb7a 100644 --- a/zppy/templates/coupled_global.py +++ b/zppy/templates/coupled_global.py @@ -3,12 +3,15 @@ import math import sys import traceback -from typing import Any, List, Tuple +from typing import Any, Dict, List, Optional, Tuple +import cftime import matplotlib as mpl import matplotlib.backends.backend_pdf import matplotlib.pyplot as plt import numpy as np +import xarray +import xcdat # noqa: F401 from netCDF4 import Dataset from readTS import TS @@ -549,11 +552,20 @@ def param_get_list(param_value): return param_value.split(",") -def set_var(exp, exp_key, var_list, valid_vars, invalid_vars, rgn): +def set_var( + exp: Dict[str, Any], + exp_key: str, + var_list: List[str], + valid_vars: List[str], + invalid_vars: List[str], + rgn: str, +) -> None: if exp[exp_key] is not None: ts = TS(exp[exp_key]) for var in var_list: try: + v: xarray.core.dataarray.DataArray + units: Optional[str] v, units = ts.globalAnnual(var) valid_vars.append(str(var)) except Exception as e: @@ -561,7 +573,7 @@ def set_var(exp, exp_key, var_list, valid_vars, invalid_vars, rgn): print(f"globalAnnual failed. Invalid var = {var}") invalid_vars.append(str(var)) continue - if len(v.shape) > 1: + if v.sizes["rgn"] > 1: # number of years x 3 regions = v.shape # 3 regions = global, northern hemisphere, southern hemisphere # We get here if we used the updated `ts` task @@ -574,18 +586,17 @@ def set_var(exp, exp_key, var_list, valid_vars, invalid_vars, rgn): n = 2 else: raise RuntimeError(f"Invalid rgn={rgn}") - v = v[:, n] # Just use nth column + v = v.isel(rgn=n) # Just use nth column elif rgn != "glb": # v only has one dimension -- glb. # Therefore it is not possible to get n or s plots. raise RuntimeError( f"var={var} only has global data. Cannot process rgn={rgn}" ) - exp["annual"][var] = v exp["annual"][var] = (v, units) if "year" not in exp["annual"]: - time = v.getTime() - exp["annual"]["year"] = [x.year for x in time.asComponentTime()] + years: np.ndarray[cftime.DatetimeNoLeap] = v.coords["time"].values + exp["annual"]["year"] = [x.year for x in years] del ts @@ -736,7 +747,7 @@ def run(parameters, rgn): # noqa: C901 or ("change_sea_level" in plots_original) ) use_ocn = plots_ocn or (not atmosphere_only and has_original_ocn_plots) - exps = [ + exps: List[Dict[str, Any]] = [ { "atmos": None if not use_atmos @@ -767,7 +778,7 @@ def run(parameters, rgn): # noqa: C901 invalid_vars: List[str] = [] # Read data - exp: Any + exp: Dict[str, Any] for exp in exps: exp["annual"] = {} @@ -845,5 +856,27 @@ def run_by_region(parameters): run(parameters, rgn) +def debug(): + args = [ + "coupled_global.py", + "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/test-346/v2.LR.historical_0201", + "v2.LR.historical_0201", + "v2_historical_0201", + "1850", + "1860", + "Blue", + "5", + "net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,change_ohc,max_moc,change_sea_level,net_atm_water_imbalance", + "False", + "None", + "None", + "LAISHA,LAISUN", + "None", + "glb,n,s", + ] + run_by_region(args) + + if __name__ == "__main__": run_by_region(sys.argv) + # debug() diff --git a/zppy/templates/readTS.py b/zppy/templates/readTS.py index a66db0e8..eda96086 100644 --- a/zppy/templates/readTS.py +++ b/zppy/templates/readTS.py @@ -1,6 +1,7 @@ from typing import Optional, Tuple import xarray +import xcdat # noqa: F401 class TS(object): @@ -8,9 +9,9 @@ def __init__(self, directory): self.directory: str = directory - # directory will be of the form `{case_dir}/post//glb/ts/monthly/{ts_num_years}yr` - self.f: xarray.core.dataset.Dataset = xarray.open_mfdataset(f"{directory}/*.nc") + # `directory` will be of the form `{case_dir}/post//glb/ts/monthly/{ts_num_years}yr` # Refactor note: `self.f = cdms2.open(filename)` gave `cdms2.dataset.Dataset` + self.f: xarray.core.dataset.Dataset = xarray.open_mfdataset(f"{directory}/*.nc") def __del__(self): @@ -20,14 +21,13 @@ def globalAnnual( self, var: str ) -> Tuple[xarray.core.dataarray.DataArray, Optional[str]]: + v: xarray.core.dataarray.DataArray units: Optional[str] = None # Constants, from AMWG diagnostics Lv = 2.501e6 Lf = 3.337e5 - v: xarray.core.dataarray.DataArray - # Is this a derived variable? if var == "RESTOM": @@ -68,16 +68,10 @@ def globalAnnual( else: # Non-derived variables - # Read variable - v = self.f.data_vars[var] - # Refactor note: `v = self.f(var)` gave `cdms2.tvariable.TransientVariable` + annual_average_dataset_for_var: xarray.core.dataset.Dataset = ( + self.f.temporal.group_average(var, "year") + ) + v = annual_average_dataset_for_var.data_vars[var] units = v.units - # Annual average - - # Refactor note: `AttributeError: 'Dataset' object has no attribute 'temporal'` seems to always occur - # Regardless if using CDAT or not, if using as object or class method. - # v = self.f.temporal.group_average(v, "year") - v = xarray.Dataset.temporal.group_average(v, "year") - return v, units From 6d520bd50a8808ae7ffeb3433879eb1b49d7adf0 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 23 May 2024 16:48:27 -0500 Subject: [PATCH 4/4] Clean up code --- .../generated/test_bundles_chrysalis.cfg | 4 +- .../generated/test_complete_run_chrysalis.cfg | 8 +-- .../generated/test_debug_chrysalis.cfg | 53 +++++++--------- ...update_bundles_expected_files_chrysalis.sh | 4 +- ...e_complete_run_expected_files_chrysalis.sh | 2 +- tests/integration/template_debug.cfg | 51 +++++++--------- tests/integration/utils.py | 2 +- zppy/templates/coupled_global.py | 61 ++++++------------- zppy/templates/e3sm_diags.bash | 4 +- zppy/templates/readTS.py | 5 +- 10 files changed, 78 insertions(+), 116 deletions(-) diff --git a/tests/integration/generated/test_bundles_chrysalis.cfg b/tests/integration/generated/test_bundles_chrysalis.cfg index 15ef3271..bd465e48 100644 --- a/tests/integration/generated/test_bundles_chrysalis.cfg +++ b/tests/integration/generated/test_bundles_chrysalis.cfg @@ -7,11 +7,11 @@ input = "/lcrc/group/e3sm/ac.forsyth2/E3SMv2/v2.LR.historical_0201" input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_bundles.py -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/test-346/v2.LR.historical_0201" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/unique_id/v2.LR.historical_0201" partition = "compute" qos = "regular" walltime = "07:00:00" -www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/test-346" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/unique_id" [bundle] diff --git a/tests/integration/generated/test_complete_run_chrysalis.cfg b/tests/integration/generated/test_complete_run_chrysalis.cfg index d3559f22..358baa50 100644 --- a/tests/integration/generated/test_complete_run_chrysalis.cfg +++ b/tests/integration/generated/test_complete_run_chrysalis.cfg @@ -7,10 +7,10 @@ input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201" input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/test-346/v2.LR.historical_0201" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201" partition = "debug" qos = "regular" -www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/test-346" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/unique_id" [climo] active = True @@ -134,7 +134,7 @@ years = "1850:1854:2", "1850:1854:4", ref_name = "v2.LR.historical_0201" ref_start_yr = 1850 ref_years = "1850-1851", - reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/test-346/v2.LR.historical_0201/post/atm/180x360_aave/clim" + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201/post/atm/180x360_aave/clim" run_type = "model_vs_model" short_ref_name = "v2.LR.historical_0201" swap_test_ref = False @@ -155,7 +155,7 @@ years = "1850:1854:2", "1850:1854:4", ref_name = "v2.LR.historical_0201" ref_start_yr = 1850 ref_years = "1850-1851", - reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/test-346/v2.LR.historical_0201/post/lnd/180x360_aave/clim" + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201/post/lnd/180x360_aave/clim" run_type = "model_vs_model" sets = "lat_lon_land", short_ref_name = "same simulation" diff --git a/tests/integration/generated/test_debug_chrysalis.cfg b/tests/integration/generated/test_debug_chrysalis.cfg index 30de023d..3e3ff988 100644 --- a/tests/integration/generated/test_debug_chrysalis.cfg +++ b/tests/integration/generated/test_debug_chrysalis.cfg @@ -12,53 +12,46 @@ environment_commands = "" input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201" input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/test-346/v2.LR.historical_0201" +# To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/unique_id/v2.LR.historical_0201" partition = "debug" qos = "regular" -www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_debug_www/test-346" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_debug_www/unique_id" [ts] active = True -e3sm_to_cmip_environment_commands = "" walltime = "00:30:00" years = "1850:1854:2", - [[ atm_monthly_glb ]] - # Note global average won't work for 3D variables. + [[ atm_monthly_180x360_aave ]] frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" - mapping_file = "glb" - years = "1850:1860:5", + ts_fmt = "cmip" - [[ lnd_monthly_glb ]] + [[ atm_daily_180x360_aave ]] + frequency = "daily" + input_files = "eam.h1" + input_subdir = "archive/atm/hist" + vars = "PRECT" + + [[ land_monthly ]] + e3sm_to_cmip_environment_commands = "" + extra_vars = "landfrac" frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" - mapping_file = "glb" vars = "LAISHA,LAISUN" - years = "1850:1860:5", + #vars = "FSH,RH2M" + ts_fmt = "cmip" -[mpas_analysis] +[ilamb] active = True -anomalyRefYear = 1850 -climo_years ="1850-1854", "1855-1860", -enso_years = "1850-1854", "1855-1860", -mesh = "EC30to60E2r2" -parallelTaskCount = 6 +grid = '180x360_aave' +nodes = 8 partition = "compute" -qos = "regular" -ts_years = "1850-1854", "1850-1860", -walltime = "00:30:00" +short_name = 'v2.LR.historical_0201' +ts_num_years = 2 +years = "1850:1854:2", -[global_time_series] -active = True -climo_years ="1850-1854", "1855-1860", -experiment_name = "v2.LR.historical_0201" -figstr = "v2_historical_0201" -moc_file=mocTimeSeries_1850-1860.nc -plots_lnd = "LAISHA,LAISUN" -ts_num_years = 5 -ts_years = "1850-1854", "1850-1860", -walltime = "00:30:00" -years = "1850-1860", + [[ land_monthly ]] diff --git a/tests/integration/generated/update_bundles_expected_files_chrysalis.sh b/tests/integration/generated/update_bundles_expected_files_chrysalis.sh index f4c4d6e4..ae3b50cb 100755 --- a/tests/integration/generated/update_bundles_expected_files_chrysalis.sh +++ b/tests/integration/generated/update_bundles_expected_files_chrysalis.sh @@ -6,9 +6,9 @@ rm -rf /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles # Your output will now become the new expectation. # Copy output so you don't have to rerun zppy to generate the output. -cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/test-346/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles +cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/unique_id/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles mkdir -p /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files -cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/test-346/v2.LR.historical_0201/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files +cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/unique_id/v2.LR.historical_0201/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files zppy_top_level=$(pwd) cd /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles # Remove the image check failures, so they don't end up in the expected files. diff --git a/tests/integration/generated/update_complete_run_expected_files_chrysalis.sh b/tests/integration/generated/update_complete_run_expected_files_chrysalis.sh index 9c63989d..0ed4a48a 100755 --- a/tests/integration/generated/update_complete_run_expected_files_chrysalis.sh +++ b/tests/integration/generated/update_complete_run_expected_files_chrysalis.sh @@ -6,7 +6,7 @@ rm -rf /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run # Your output will now become the new expectation. # Copy output so you don't have to rerun zppy to generate the output. -cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/test-346/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run +cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/unique_id/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run zppy_top_level=$(pwd) cd /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run # Remove the image check failures, so they don't end up in the expected files. diff --git a/tests/integration/template_debug.cfg b/tests/integration/template_debug.cfg index baded2b0..d9a64bf5 100644 --- a/tests/integration/template_debug.cfg +++ b/tests/integration/template_debug.cfg @@ -12,53 +12,46 @@ environment_commands = "#expand environment_commands#" input = "#expand user_input#/E3SMv2/v2.LR.historical_0201" input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" +# To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py output = "#expand user_output#zppy_test_debug_output/#expand unique_id#/v2.LR.historical_0201" partition = "#expand partition_short#" qos = "#expand qos_short#" -www = "#expand user_www#zppy_test_debug_www/#expand unique_id#" +www = "#expand user_www#zppy_debug_www/#expand unique_id#" [ts] active = True -e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" walltime = "00:30:00" years = "1850:1854:2", - [[ atm_monthly_glb ]] - # Note global average won't work for 3D variables. + [[ atm_monthly_180x360_aave ]] frequency = "monthly" input_files = "eam.h0" input_subdir = "archive/atm/hist" - mapping_file = "glb" - years = "1850:1860:5", + ts_fmt = "cmip" - [[ lnd_monthly_glb ]] + [[ atm_daily_180x360_aave ]] + frequency = "daily" + input_files = "eam.h1" + input_subdir = "archive/atm/hist" + vars = "PRECT" + + [[ land_monthly ]] + e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" + extra_vars = "landfrac" frequency = "monthly" input_files = "elm.h0" input_subdir = "archive/lnd/hist" - mapping_file = "glb" vars = "LAISHA,LAISUN" - years = "1850:1860:5", + #vars = "FSH,RH2M" + ts_fmt = "cmip" -[mpas_analysis] +[ilamb] active = True -anomalyRefYear = 1850 -climo_years ="1850-1854", "1855-1860", -enso_years = "1850-1854", "1855-1860", -mesh = "EC30to60E2r2" -parallelTaskCount = 6 +grid = '180x360_aave' +nodes = 8 partition = "#expand partition_long#" -qos = "#expand qos_long#" -ts_years = "1850-1854", "1850-1860", -walltime = "#expand mpas_analysis_walltime#" +short_name = 'v2.LR.historical_0201' +ts_num_years = 2 +years = "1850:1854:2", -[global_time_series] -active = True -climo_years ="1850-1854", "1855-1860", -experiment_name = "v2.LR.historical_0201" -figstr = "v2_historical_0201" -moc_file=mocTimeSeries_1850-1860.nc -plots_lnd = "LAISHA,LAISUN" -ts_num_years = 5 -ts_years = "1850-1854", "1850-1860", -walltime = "00:30:00" -years = "1850-1860", + [[ land_monthly ]] diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 9f4282fc..b8f1e74c 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -7,7 +7,7 @@ from mache import MachineInfo from PIL import Image, ImageChops, ImageDraw -UNIQUE_ID = "test-346" +UNIQUE_ID = "unique_id" # Image checking ########################################################## diff --git a/zppy/templates/coupled_global.py b/zppy/templates/coupled_global.py index f338eb7a..762e7724 100644 --- a/zppy/templates/coupled_global.py +++ b/zppy/templates/coupled_global.py @@ -11,7 +11,6 @@ import matplotlib.pyplot as plt import numpy as np import xarray -import xcdat # noqa: F401 from netCDF4 import Dataset from readTS import TS @@ -586,7 +585,7 @@ def set_var( n = 2 else: raise RuntimeError(f"Invalid rgn={rgn}") - v = v.isel(rgn=n) # Just use nth column + v = v.isel(rgn=n) # Just use nth region elif rgn != "glb": # v only has one dimension -- glb. # Therefore it is not possible to get n or s plots. @@ -749,24 +748,24 @@ def run(parameters, rgn): # noqa: C901 use_ocn = plots_ocn or (not atmosphere_only and has_original_ocn_plots) exps: List[Dict[str, Any]] = [ { - "atmos": None - if not use_atmos - else "{}/post/atm/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years), - "ice": None - if not plots_ice - else "{}/post/ice/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years), - "land": None - if not plots_lnd - else "{}/post/lnd/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years), - "ocean": None - if not use_ocn - else "{}/post/ocn/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years), - "moc": None - if not use_ocn - else "{}/post/ocn/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years), - "vol": None - if not use_ocn - else "{}/post/ocn/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years), + "atmos": f"{case_dir}/post/atm/glb/ts/monthly/{ts_num_years}yr/" + if use_atmos + else None, + "ice": f"{case_dir}/post/ice/glb/ts/monthly/{ts_num_years}yr/" + if plots_ice + else None, + "land": f"{case_dir}/post/lnd/glb/ts/monthly/{ts_num_years}yr/" + if plots_lnd + else None, + "ocean": f"{case_dir}/post/ocn/glb/ts/monthly/{ts_num_years}yr/" + if use_ocn + else None, + "moc": f"{case_dir}/post/ocn/glb/ts/monthly/{ts_num_years}yr/" + if use_ocn + else None, + "vol": f"{case_dir}/post/ocn/glb/ts/monthly/{ts_num_years}yr/" + if use_ocn + else None, "name": experiment_name, "yoffset": 0.0, "yr": ([year1, year2],), @@ -856,27 +855,5 @@ def run_by_region(parameters): run(parameters, rgn) -def debug(): - args = [ - "coupled_global.py", - "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/test-346/v2.LR.historical_0201", - "v2.LR.historical_0201", - "v2_historical_0201", - "1850", - "1860", - "Blue", - "5", - "net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,change_ohc,max_moc,change_sea_level,net_atm_water_imbalance", - "False", - "None", - "None", - "LAISHA,LAISUN", - "None", - "glb,n,s", - ] - run_by_region(args) - - if __name__ == "__main__": run_by_region(sys.argv) - # debug() diff --git a/zppy/templates/e3sm_diags.bash b/zppy/templates/e3sm_diags.bash index 4f70005b..4823e0ff 100644 --- a/zppy/templates/e3sm_diags.bash +++ b/zppy/templates/e3sm_diags.bash @@ -108,7 +108,7 @@ create_links_ts() # xml file will cover the whole period from year1 to year2 xml_name=${v}_${begin_year}01_${end_year}12.xml export CDMS_NO_MPI=true - cdscan ${xml_name} ${v}_files.txt + cdscan -x ${xml_name} -f ${v}_files.txt if [ $? != 0 ]; then cd {{ scriptDir }} echo "ERROR (${error_num})" > {{ prefix }}.status @@ -129,7 +129,7 @@ create_links_ts_rof() cd ${ts_rof_dir_destination} v="RIVER_DISCHARGE_OVER_LAND_LIQ" xml_name=${v}_${begin_year}01_${end_year}12.xml - cdscan ${xml_name} ${ts_rof_dir_source}/${v}_*.nc + cdscan -x ${xml_name} ${ts_rof_dir_source}/${v}_*.nc if [ $? != 0 ]; then cd {{ scriptDir }} echo "ERROR (${error_num})" > {{ prefix }}.status diff --git a/zppy/templates/readTS.py b/zppy/templates/readTS.py index eda96086..312e604d 100644 --- a/zppy/templates/readTS.py +++ b/zppy/templates/readTS.py @@ -9,9 +9,8 @@ def __init__(self, directory): self.directory: str = directory - # `directory` will be of the form `{case_dir}/post//glb/ts/monthly/{ts_num_years}yr` - # Refactor note: `self.f = cdms2.open(filename)` gave `cdms2.dataset.Dataset` - self.f: xarray.core.dataset.Dataset = xarray.open_mfdataset(f"{directory}/*.nc") + # `directory` will be of the form `{case_dir}/post//glb/ts/monthly/{ts_num_years}yr/` + self.f: xarray.core.dataset.Dataset = xarray.open_mfdataset(f"{directory}*.nc") def __del__(self):