Skip to content

Commit cb70031

Browse files
schlunmaschlunma
committed
Added tests for input/output filenames for ICON and EMAC CMORizer
1 parent 62b5c30 commit cb70031

File tree

4 files changed

+235
-8
lines changed

4 files changed

+235
-8
lines changed

‎esmvalcore/_config/extra_facets/ipslcm-mappings.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ IPSL-CM6:
111111
# -> general variables
112112
precip: {ipsl_varname: precip, <<: *atmvars}
113113
slp: {ipsl_varname: slp, <<: *atmvars}
114-
114+
115115
# -> Turbulent fluxes
116116
taux: {ipsl_varname: taux, <<: *atmvars}
117117
tauy: {ipsl_varname: tauy, <<: *atmvars}
@@ -136,8 +136,8 @@ IPSL-CM6:
136136
SWdnSFC: {ipsl_varname: SWdnSFC, <<: *atmvars}
137137
LWdnSFcclr: {ipsl_varname: LWdnSFcclr, <<: *atmvars}
138138
SWdnSFcclr: {ipsl_varname: SWdnSFcclr, <<: *atmvars}
139-
140-
139+
140+
141141
# =================================================
142142
Lmon:
143143
# ===============================================

‎esmvalcore/config-developer.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ EMAC:
133133
default: '{exp}/{channel}'
134134
input_file:
135135
default: '{exp}*{channel}{postproc_flag}.nc'
136-
output_file: '{dataset}_{exp}_{channel}_{mip}_{short_name}'
136+
output_file: '{project}_{dataset}_{exp}_{channel}_{mip}_{short_name}'
137137
cmor_type: 'CMIP6'
138138
ignore_warnings:
139139
load:

‎tests/integration/data_finder.yml

+213-4
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,77 @@ get_output_file:
2626
preproc_dir: /test
2727
output_file: /test/test_diag/test/CMIP5_HadGEM2-ES_Amon_historical-rcp85_r1i1p1_ta_1960-1980.nc
2828

29+
# EMAC
30+
31+
- variable:
32+
variable_group: test
33+
short_name: tas
34+
original_short_name: tas
35+
dataset: EMAC
36+
project: EMAC
37+
cmor_table: CMIP6
38+
frequency: mon
39+
mip: Amon
40+
exp: amip
41+
timerange: '1960/1980'
42+
diagnostic: test_diag
43+
preprocessor: test_preproc
44+
preproc_dir: this/is/a/path
45+
output_file: this/is/a/path/test_diag/test/EMAC_EMAC_amip_Amon_Amon_tas_1960-1980.nc
46+
47+
- variable:
48+
variable_group: test
49+
short_name: tas
50+
original_short_name: tas
51+
dataset: EMAC
52+
project: EMAC
53+
cmor_table: CMIP6
54+
frequency: mon
55+
mip: Amon
56+
exp: piControl
57+
channel: CH
58+
postproc_flag: -p-mm
59+
timerange: '199001/199002'
60+
diagnostic: test_diag
61+
preprocessor: test_preproc
62+
preproc_dir: this/is/a/path
63+
output_file: this/is/a/path/test_diag/test/EMAC_EMAC_piControl_CH_Amon_tas_199001-199002.nc
64+
65+
# ICON
66+
67+
- variable:
68+
variable_group: test
69+
short_name: tas
70+
original_short_name: tas
71+
dataset: ICON
72+
project: ICON
73+
cmor_table: CMIP6
74+
frequency: mon
75+
mip: Amon
76+
exp: amip
77+
timerange: '1960/1980'
78+
diagnostic: test_diag
79+
preprocessor: test_preproc
80+
preproc_dir: this/is/a/path
81+
output_file: this/is/a/path/test_diag/test/ICON_ICON_amip_atm_2d_ml_Amon_tas_1960-1980.nc
82+
83+
- variable:
84+
variable_group: test
85+
short_name: tas
86+
original_short_name: tas
87+
dataset: ICON
88+
project: ICON
89+
cmor_table: CMIP6
90+
frequency: mon
91+
mip: Amon
92+
exp: amip
93+
var_type: custom_var_type
94+
timerange: '20000101/20000102'
95+
diagnostic: test_diag
96+
preprocessor: test_preproc
97+
preproc_dir: this/is/a/path
98+
output_file: this/is/a/path/test_diag/test/ICON_ICON_amip_custom_var_type_Amon_tas_20000101-20000102.nc
99+
29100

30101
get_input_filelist:
31102
- drs: default
@@ -460,9 +531,9 @@ get_input_filelist:
460531
file_patterns:
461532
- simulation_*_t2m.nc
462533
- simulation_*_histmth.nc
463-
found_files:
534+
found_files:
464535
- thredds/tgcc/store/p86caub/IPSLCM6/PROD/historical/simulation/ATM/Output/MO/simulation_18500101_18591231_1M_histmth.nc
465-
536+
466537
- drs: default
467538
variable:
468539
<<: *ipsl_variable
@@ -475,9 +546,9 @@ get_input_filelist:
475546
file_patterns:
476547
- simulation_*_t2m.nc
477548
- simulation_*_histmth.nc
478-
found_files:
549+
found_files:
479550
- thredds/tgcc/store/p86caub/IPSLCM6/PROD/historical/simulation/ATM/Analyse/TS_MO/simulation_18500101_20141231_1M_t2m.nc
480-
551+
481552
# Test fx files
482553

483554
- drs: default
@@ -779,3 +850,141 @@ get_input_filelist:
779850
file_patterns:
780851
- OBS6_ERA-Interim_reanaly_42_Omon_deptho[_.]*nc
781852
found_files: []
853+
854+
# EMAC
855+
856+
- drs: default
857+
variable:
858+
variable_group: test
859+
short_name: tas
860+
original_short_name: tas
861+
dataset: EMAC
862+
project: EMAC
863+
cmor_table: CMIP6
864+
frequency: mon
865+
mip: Amon
866+
exp: amip
867+
timerange: '200002/200003'
868+
diagnostic: test_diag
869+
preprocessor: test_preproc
870+
available_files:
871+
- amip/Amon/amip___________200001_Amon.nc
872+
- amip/Amon/amip___________200002_Amon.nc
873+
- amip/Amon/amip___________200003_Amon.nc
874+
- amip/Amon/amip___________200001_Amon-p-mm.nc
875+
- amip/Amon/amip___________200002_Amon-p-mm.nc
876+
- amip/Amon/amip___________200003_Amon-p-mm.nc
877+
- amip/rad/amip___________200001_rad.nc
878+
- amip/rad/amip___________200002_rad.nc
879+
- amip/rad/amip___________200003_rad.nc
880+
- amip/rad/amip___________200001_rad-p-mm.nc
881+
- amip/rad/amip___________200002_rad-p-mm.nc
882+
- amip/rad/amip___________200003_rad-p-mm.nc
883+
dirs:
884+
- amip/Amon
885+
file_patterns:
886+
- amip*Amon.nc
887+
found_files:
888+
- amip/Amon/amip___________200002_Amon.nc
889+
- amip/Amon/amip___________200003_Amon.nc
890+
891+
- drs: default
892+
variable:
893+
variable_group: test
894+
short_name: tas
895+
original_short_name: tas
896+
dataset: EMAC
897+
project: EMAC
898+
cmor_table: CMIP6
899+
frequency: mon
900+
mip: Amon
901+
exp: amip
902+
channel: rad
903+
postproc_flag: -p-mm
904+
timerange: '200001/200002'
905+
diagnostic: test_diag
906+
preprocessor: test_preproc
907+
available_files:
908+
- amip/Amon/amip___________200001_Amon.nc
909+
- amip/Amon/amip___________200002_Amon.nc
910+
- amip/Amon/amip___________200003_Amon.nc
911+
- amip/Amon/amip___________200001_Amon-p-mm.nc
912+
- amip/Amon/amip___________200002_Amon-p-mm.nc
913+
- amip/Amon/amip___________200003_Amon-p-mm.nc
914+
- amip/rad/amip___________200001_rad.nc
915+
- amip/rad/amip___________200002_rad.nc
916+
- amip/rad/amip___________200003_rad.nc
917+
- amip/rad/amip___________200001_rad-p-mm.nc
918+
- amip/rad/amip___________200002_rad-p-mm.nc
919+
- amip/rad/amip___________200003_rad-p-mm.nc
920+
dirs:
921+
- amip/rad
922+
file_patterns:
923+
- amip*rad-p-mm.nc
924+
found_files:
925+
- amip/rad/amip___________200001_rad-p-mm.nc
926+
- amip/rad/amip___________200002_rad-p-mm.nc
927+
928+
# ICON
929+
930+
- drs: default
931+
variable:
932+
variable_group: test
933+
short_name: tas
934+
original_short_name: tas
935+
dataset: ICON
936+
project: ICON
937+
cmor_table: CMIP6
938+
frequency: mon
939+
mip: Amon
940+
exp: amip
941+
timerange: '200002/200003'
942+
diagnostic: test_diag
943+
preprocessor: test_preproc
944+
available_files:
945+
- amip/amip_atm_2d_ml_20000101T000000Z.nc
946+
- amip/amip_atm_2d_ml_20000201T000000Z.nc
947+
- amip/amip_atm_2d_ml_20000301T000000Z.nc
948+
- amip/outdata/amip_atm_2d_ml_20000401T000000Z.nc
949+
- amip/outdata/amip_atm_2d_ml_20000501T000000Z.nc
950+
- amip/outdata/amip_atm_2d_ml_20000601T000000Z.nc
951+
dirs:
952+
- amip
953+
- amip/outdata
954+
file_patterns:
955+
- amip_atm_2d_ml*.nc
956+
found_files:
957+
- amip/amip_atm_2d_ml_20000201T000000Z.nc
958+
- amip/amip_atm_2d_ml_20000301T000000Z.nc
959+
960+
- drs: default
961+
variable:
962+
variable_group: test
963+
short_name: tas
964+
original_short_name: tas
965+
dataset: ICON
966+
project: ICON
967+
cmor_table: CMIP6
968+
frequency: mon
969+
mip: Amon
970+
exp: amip
971+
var_type: var
972+
timerange: '200003/200005'
973+
diagnostic: test_diag
974+
preprocessor: test_preproc
975+
available_files:
976+
- amip/amip_var_20000101T000000Z.nc
977+
- amip/amip_var_20000201T000000Z.nc
978+
- amip/amip_var_20000301T000000Z.nc
979+
- amip/outdata/amip_var_20000401T000000Z.nc
980+
- amip/outdata/amip_var_20000501T000000Z.nc
981+
- amip/outdata/amip_var_20000601T000000Z.nc
982+
dirs:
983+
- amip
984+
- amip/outdata
985+
file_patterns:
986+
- amip_var*.nc
987+
found_files:
988+
- amip/amip_var_20000301T000000Z.nc
989+
- amip/outdata/amip_var_20000401T000000Z.nc
990+
- amip/outdata/amip_var_20000501T000000Z.nc

‎tests/integration/test_data_finder.py

+18
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,20 @@
2525
CONFIG = yaml.safe_load(file)
2626

2727

28+
def _augment_with_extra_facets(variable):
29+
"""Augment variable dict with extra facets."""
30+
extra_facets = esmvalcore._config.get_extra_facets(
31+
variable['project'],
32+
variable['dataset'],
33+
variable['mip'],
34+
variable['short_name'],
35+
(),
36+
)
37+
for (key, val) in extra_facets.items():
38+
if key not in variable:
39+
variable[key] = val
40+
41+
2842
def print_path(path):
2943
"""Print path."""
3044
txt = path
@@ -68,6 +82,7 @@ def create_tree(path, filenames=None, symlinks=None):
6882
@pytest.mark.parametrize('cfg', CONFIG['get_output_file'])
6983
def test_get_output_file(cfg):
7084
"""Test getting output name for preprocessed files."""
85+
_augment_with_extra_facets(cfg['variable'])
7186
output_file = get_output_file(cfg['variable'], cfg['preproc_dir'])
7287
assert output_file == cfg['output_file']
7388

@@ -88,6 +103,9 @@ def test_get_input_filelist(root, cfg):
88103
create_tree(root, cfg.get('available_files'),
89104
cfg.get('available_symlinks'))
90105

106+
# Augment variable dict with extra facets
107+
_augment_with_extra_facets(cfg['variable'])
108+
91109
# Find files
92110
rootpath = {cfg['variable']['project']: [root]}
93111
drs = {cfg['variable']['project']: cfg['drs']}

0 commit comments

Comments
 (0)