Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace CDAT #519

Merged
merged 4 commits into from
May 23, 2024
Merged

Replace CDAT #519

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ea853bd
Replace CDAT
forsyth2 Oct 3, 2023
05e432c
types without xml
forsyth2 May 14, 2024
0c233d7
glb ts without cdat
forsyth2 May 16, 2024
6d520bd
Clean up code
forsyth2 May 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions tests/integration/generated/test_bundles_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@ input = "/lcrc/group/e3sm/ac.forsyth2/E3SMv2/v2.LR.historical_0201"
input_subdir = archive/atm/hist
mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
# To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_bundles.py
output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/unique_id/v2.LR.historical_0201"
output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/test-346/v2.LR.historical_0201"
partition = "compute"
qos = "regular"
walltime = "07:00:00"
www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/unique_id"
www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/test-346"

[bundle]

Expand Down
8 changes: 4 additions & 4 deletions tests/integration/generated/test_complete_run_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@ input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201"
input_subdir = archive/atm/hist
mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
# To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py
output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201"
output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/test-346/v2.LR.historical_0201"
partition = "debug"
qos = "regular"
www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/unique_id"
www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/test-346"

[climo]
active = True
Expand Down Expand Up @@ -134,7 +134,7 @@ years = "1850:1854:2", "1850:1854:4",
ref_name = "v2.LR.historical_0201"
ref_start_yr = 1850
ref_years = "1850-1851",
reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201/post/atm/180x360_aave/clim"
reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/test-346/v2.LR.historical_0201/post/atm/180x360_aave/clim"
run_type = "model_vs_model"
short_ref_name = "v2.LR.historical_0201"
swap_test_ref = False
Expand All @@ -155,7 +155,7 @@ years = "1850:1854:2", "1850:1854:4",
ref_name = "v2.LR.historical_0201"
ref_start_yr = 1850
ref_years = "1850-1851",
reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201/post/lnd/180x360_aave/clim"
reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/test-346/v2.LR.historical_0201/post/lnd/180x360_aave/clim"
run_type = "model_vs_model"
sets = "lat_lon_land",
short_ref_name = "same simulation"
Expand Down
53 changes: 30 additions & 23 deletions tests/integration/generated/test_debug_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,46 +12,53 @@ environment_commands = ""
input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201"
input_subdir = archive/atm/hist
mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
# To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py
output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/unique_id/v2.LR.historical_0201"
output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/test-346/v2.LR.historical_0201"
partition = "debug"
qos = "regular"
www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_debug_www/unique_id"
www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_debug_www/test-346"

[ts]
active = True
e3sm_to_cmip_environment_commands = ""
walltime = "00:30:00"
years = "1850:1854:2",

[[ atm_monthly_180x360_aave ]]
[[ atm_monthly_glb ]]
# Note global average won't work for 3D variables.
frequency = "monthly"
input_files = "eam.h0"
input_subdir = "archive/atm/hist"
ts_fmt = "cmip"
mapping_file = "glb"
years = "1850:1860:5",

[[ atm_daily_180x360_aave ]]
frequency = "daily"
input_files = "eam.h1"
input_subdir = "archive/atm/hist"
vars = "PRECT"

[[ land_monthly ]]
e3sm_to_cmip_environment_commands = ""
extra_vars = "landfrac"
[[ lnd_monthly_glb ]]
frequency = "monthly"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
mapping_file = "glb"
vars = "LAISHA,LAISUN"
#vars = "FSH,RH2M"
ts_fmt = "cmip"
years = "1850:1860:5",

[ilamb]
[mpas_analysis]
active = True
grid = '180x360_aave'
nodes = 8
anomalyRefYear = 1850
climo_years ="1850-1854", "1855-1860",
enso_years = "1850-1854", "1855-1860",
mesh = "EC30to60E2r2"
parallelTaskCount = 6
partition = "compute"
short_name = 'v2.LR.historical_0201'
ts_num_years = 2
years = "1850:1854:2",
qos = "regular"
ts_years = "1850-1854", "1850-1860",
walltime = "00:30:00"

[[ land_monthly ]]
[global_time_series]
active = True
climo_years ="1850-1854", "1855-1860",
experiment_name = "v2.LR.historical_0201"
figstr = "v2_historical_0201"
moc_file=mocTimeSeries_1850-1860.nc
plots_lnd = "LAISHA,LAISUN"
ts_num_years = 5
ts_years = "1850-1854", "1850-1860",
walltime = "00:30:00"
years = "1850-1860",
4 changes: 2 additions & 2 deletions tests/integration/generated/update_bundles_expected_files_chrysalis.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@
rm -rf /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles
# Your output will now become the new expectation.
# Copy output so you don't have to rerun zppy to generate the output.
cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/unique_id/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles
cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/test-346/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles
mkdir -p /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files
cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/unique_id/v2.LR.historical_0201/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files
cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/test-346/v2.LR.historical_0201/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files
zppy_top_level=$(pwd)
cd /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles
# Remove the image check failures, so they don't end up in the expected files.
Expand Down
2 changes: 1 addition & 1 deletion tests/integration/generated/update_complete_run_expected_files_chrysalis.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
rm -rf /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
# Your output will now become the new expectation.
# Copy output so you don't have to rerun zppy to generate the output.
cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/unique_id/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/test-346/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
zppy_top_level=$(pwd)
cd /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
# Remove the image check failures, so they don't end up in the expected files.
Expand Down
51 changes: 29 additions & 22 deletions tests/integration/template_debug.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,46 +12,53 @@ environment_commands = "#expand environment_commands#"
input = "#expand user_input#/E3SMv2/v2.LR.historical_0201"
input_subdir = archive/atm/hist
mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
# To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py
output = "#expand user_output#zppy_test_debug_output/#expand unique_id#/v2.LR.historical_0201"
partition = "#expand partition_short#"
qos = "#expand qos_short#"
www = "#expand user_www#zppy_debug_www/#expand unique_id#"
www = "#expand user_www#zppy_test_debug_www/#expand unique_id#"

[ts]
active = True
e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#"
walltime = "00:30:00"
years = "1850:1854:2",

[[ atm_monthly_180x360_aave ]]
[[ atm_monthly_glb ]]
# Note global average won't work for 3D variables.
frequency = "monthly"
input_files = "eam.h0"
input_subdir = "archive/atm/hist"
ts_fmt = "cmip"
mapping_file = "glb"
years = "1850:1860:5",

[[ atm_daily_180x360_aave ]]
frequency = "daily"
input_files = "eam.h1"
input_subdir = "archive/atm/hist"
vars = "PRECT"

[[ land_monthly ]]
e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#"
extra_vars = "landfrac"
[[ lnd_monthly_glb ]]
frequency = "monthly"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
mapping_file = "glb"
vars = "LAISHA,LAISUN"
#vars = "FSH,RH2M"
ts_fmt = "cmip"
years = "1850:1860:5",

[ilamb]
[mpas_analysis]
active = True
grid = '180x360_aave'
nodes = 8
anomalyRefYear = 1850
climo_years ="1850-1854", "1855-1860",
enso_years = "1850-1854", "1855-1860",
mesh = "EC30to60E2r2"
parallelTaskCount = 6
partition = "#expand partition_long#"
short_name = 'v2.LR.historical_0201'
ts_num_years = 2
years = "1850:1854:2",
qos = "#expand qos_long#"
ts_years = "1850-1854", "1850-1860",
walltime = "#expand mpas_analysis_walltime#"

[[ land_monthly ]]
[global_time_series]
active = True
climo_years ="1850-1854", "1855-1860",
experiment_name = "v2.LR.historical_0201"
figstr = "v2_historical_0201"
moc_file=mocTimeSeries_1850-1860.nc
plots_lnd = "LAISHA,LAISUN"
ts_num_years = 5
ts_years = "1850-1854", "1850-1860",
walltime = "00:30:00"
years = "1850-1860",
2 changes: 1 addition & 1 deletion tests/integration/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from mache import MachineInfo
from PIL import Image, ImageChops, ImageDraw

UNIQUE_ID = "unique_id"
UNIQUE_ID = "test-346"

# Image checking ##########################################################

Expand Down
20 changes: 5 additions & 15 deletions zppy/templates/coupled_global.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -740,32 +740,22 @@ def run(parameters, rgn): # noqa: C901
{
"atmos": None
if not use_atmos
else "{}/post/atm/glb/ts/monthly/{}yr/glb.xml".format(
case_dir, ts_num_years
),
else "{}/post/atm/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years),
"ice": None
if not plots_ice
else "{}/post/ice/glb/ts/monthly/{}yr/glb.xml".format(
case_dir, ts_num_years
),
else "{}/post/ice/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years),
"land": None
if not plots_lnd
else "{}/post/lnd/glb/ts/monthly/{}yr/glb.xml".format(
case_dir, ts_num_years
),
else "{}/post/lnd/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years),
"ocean": None
if not use_ocn
else "{}/post/ocn/glb/ts/monthly/{}yr/glb.xml".format(
case_dir, ts_num_years
),
else "{}/post/ocn/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years),
"moc": None
if not use_ocn
else "{}/post/ocn/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years),
"vol": None
if not use_ocn
else "{}/post/ocn/glb/ts/monthly/{}yr/glb.xml".format(
case_dir, ts_num_years
),
else "{}/post/ocn/glb/ts/monthly/{}yr/".format(case_dir, ts_num_years),
"name": experiment_name,
"yoffset": 0.0,
"yr": ([year1, year2],),
Expand Down
4 changes: 2 additions & 2 deletions zppy/templates/e3sm_diags.bash
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ create_links_ts()
# xml file will cover the whole period from year1 to year2
xml_name=${v}_${begin_year}01_${end_year}12.xml
export CDMS_NO_MPI=true
cdscan -x ${xml_name} -f ${v}_files.txt
cdscan ${xml_name} ${v}_files.txt
if [ $? != 0 ]; then
cd {{ scriptDir }}
echo "ERROR (${error_num})" > {{ prefix }}.status
Expand All @@ -129,7 +129,7 @@ create_links_ts_rof()
cd ${ts_rof_dir_destination}
v="RIVER_DISCHARGE_OVER_LAND_LIQ"
xml_name=${v}_${begin_year}01_${end_year}12.xml
cdscan -x ${xml_name} ${ts_rof_dir_source}/${v}_*.nc
cdscan ${xml_name} ${ts_rof_dir_source}/${v}_*.nc
if [ $? != 0 ]; then
cd {{ scriptDir }}
echo "ERROR (${error_num})" > {{ prefix }}.status
Expand Down
31 changes: 0 additions & 31 deletions zppy/templates/global_time_series.bash
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,28 +39,7 @@ global_ts_dir={{ global_time_series_dir }}
export CDMS_NO_MPI=true

use_atm={{ use_atm }}
if [[ ${use_atm,,} == "true" ]]; then
echo 'Create xml files for atm'
cd ${case_dir}/post/atm/glb/ts/monthly/${ts_num_years}yr
cdscan -x glb.xml *.nc
if [ $? != 0 ]; then
cd {{ scriptDir }}
echo 'ERROR (1)' > {{ prefix }}.status
exit 1
fi
fi

use_lnd={{ use_lnd }}
if [[ ${use_lnd,,} == "true" ]]; then
echo 'Create xml files for lnd'
cd ${case_dir}/post/lnd/glb/ts/monthly/${ts_num_years}yr
cdscan -x glb.xml *.nc
if [ $? != 0 ]; then
cd {{ scriptDir }}
echo 'ERROR (2)' > {{ prefix }}.status
exit 2
fi
fi

use_ocn={{ use_ocn }}
if [[ ${use_ocn,,} == "true" ]]; then
Expand All @@ -75,16 +54,6 @@ if [[ ${use_ocn,,} == "true" ]]; then
exit 3
fi

echo 'Create xml for for ocn'
export CDMS_NO_MPI=true
cd ${case_dir}/post/ocn/glb/ts/monthly/${ts_num_years}yr
cdscan -x glb.xml mpaso.glb*.nc
if [ $? != 0 ]; then
cd {{ scriptDir }}
echo 'ERROR (4)' > {{ prefix }}.status
exit 4
fi

echo 'Copy moc file'
cd ${case_dir}/post/analysis/mpas_analysis/cache/timeseries/moc
cp ${moc_file} ../../../../../ocn/glb/ts/monthly/${ts_num_years}yr/
Expand Down
30 changes: 21 additions & 9 deletions zppy/templates/readTS.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,26 +1,33 @@
import cdms2
import cdutil
from typing import Optional, Tuple

import xarray


class TS(object):
def __init__(self, filename):
def __init__(self, directory):

self.filename = filename
self.directory: str = directory

self.f = cdms2.open(filename)
# directory will be of the form `{case_dir}/post/<componen>/glb/ts/monthly/{ts_num_years}yr`
self.f: xarray.core.dataset.Dataset = xarray.open_mfdataset(f"{directory}/*.nc")
# Refactor note: `self.f = cdms2.open(filename)` gave `cdms2.dataset.Dataset`

def __del__(self):

self.f.close()

def globalAnnual(self, var):
def globalAnnual(
self, var: str
) -> Tuple[xarray.core.dataarray.DataArray, Optional[str]]:

units = None
units: Optional[str] = None

# Constants, from AMWG diagnostics
Lv = 2.501e6
Lf = 3.337e5

v: xarray.core.dataarray.DataArray

# Is this a derived variable?
if var == "RESTOM":

Expand Down Expand Up @@ -62,10 +69,15 @@ def globalAnnual(self, var):
# Non-derived variables

# Read variable
v = self.f(var)
v = self.f.data_vars[var]
# Refactor note: `v = self.f(var)` gave `cdms2.tvariable.TransientVariable`
units = v.units

# Annual average
v = cdutil.YEAR(v)

# Refactor note: `AttributeError: 'Dataset' object has no attribute 'temporal'` seems to always occur
# Regardless if using CDAT or not, if using as object or class method.
# v = self.f.temporal.group_average(v, "year")
v = xarray.Dataset.temporal.group_average(v, "year")

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tomvothecoder It does indeed seem like I'm able to eliminate the cdscan call to generate an xml + the self.f = cdms2.open(filename) line by using xarray.open_mfdataset(f"{directory}/*.nc")

However, no matter what I do, Python doesn't seem to think temporal exists. I'm looking at https://xcdat.readthedocs.io/en/latest/generated/xarray.Dataset.temporal.group_average.html, and this is running in the latest Unified environment. Am I using the wrong averaging function here? (This line replaces v = cdutil.YEAR(v))

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Make sure to import xcdat to gain access to the .temporal accessor class from xCDAT.

The xCDAT accessor classes extend top of xarray.Dataset objects.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh I see, I just assumed it would be an xarray sub-module. Thanks for the info!

return v, units
Loading