Skip to content

Commit c3d4dde

Browse files
author
Javier Vegas-Regidor
committed
Add install and fix some scripts
1 parent eda9cc6 commit c3d4dde

30 files changed

+156
-87
lines changed

‎esmvaltool/cmorizers/obs/cmorize_obs.py

+57-19
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import logging
1717
import os
1818
import subprocess
19+
import shutil
1920
from pathlib import Path
2021

2122
import esmvalcore
@@ -160,7 +161,7 @@ def download_dataset(self, dataset, start_date, end_date, overwrite):
160161
self.config, dataset, start_date, end_date, overwrite)
161162
logger.info('%s downloaded', dataset)
162163

163-
def format(self):
164+
def format(self, start, end, install):
164165
"""Format all available datasets."""
165166
logger.info("Running the CMORization scripts.")
166167
# datsets dictionary of Tier keys
@@ -174,13 +175,12 @@ def format(self):
174175
failed_datasets = []
175176
for tier in datasets:
176177
for dataset in datasets:
177-
if not self.format_dataset(dataset):
178+
if not self.format_dataset(dataset, start, end, install):
178179
failed_datasets.append(dataset)
179180

180181
if failed_datasets:
181182
raise Exception(
182-
'Could not find cmorizers for %s datasets ' %
183-
' '.join(failed_datasets)
183+
f'Format failed for datasets {" ".join(failed_datasets)}'
184184
)
185185

186186
def _assemble_datasets(self):
@@ -205,7 +205,7 @@ def _assemble_datasets(self):
205205

206206
return datasets
207207

208-
def format_dataset(self, dataset):
208+
def format_dataset(self, dataset, start, end, install):
209209
"""
210210
Format a single dataset.
211211
@@ -239,16 +239,37 @@ def format_dataset(self, dataset):
239239
logger.info("Reformat script: %s", reformat_script_root)
240240
if os.path.isfile(reformat_script_root + '.ncl'):
241241
reformat_script = reformat_script_root + '.ncl'
242-
return self._run_ncl_script(
242+
success = self._run_ncl_script(
243243
in_data_dir,
244244
out_data_dir,
245245
dataset,
246246
reformat_script,
247+
start,
248+
end
247249
)
248250
elif os.path.isfile(reformat_script_root + '.py'):
249-
return self._run_pyt_script(in_data_dir, out_data_dir, dataset)
250-
logger.error('Could not find cmorizer for %s', dataset)
251-
return False
251+
success = self._run_pyt_script(
252+
in_data_dir, out_data_dir, dataset, start, end)
253+
else:
254+
logger.error('Could not find formatter for %s', dataset)
255+
return False
256+
if not success:
257+
logger.error('Foramtting failed for datset %s', dataset)
258+
return False
259+
if install:
260+
rootpath = self.config['rootpath']
261+
target_dir = rootpath.get('OBS', rootpath['default'])[0]
262+
target_dir = os.path.join(target_dir, tier, dataset)
263+
if os.path.isdir(target_dir):
264+
logger.info(
265+
'Automatic installation of dataset %s skipped: '
266+
'target folder %s already exists',
267+
dataset, target_dir)
268+
else:
269+
logger.info(
270+
'Installing dataset %s in folder %s', dataset, target_dir)
271+
shutil.copytree(out_data_dir, target_dir)
272+
return True
252273

253274
def _get_dataset_tier(self, dataset):
254275
for tier in [2, 3]:
@@ -258,15 +279,25 @@ def _get_dataset_tier(self, dataset):
258279
return None
259280

260281
def _write_ncl_settings(self, project_info, dataset, run_dir,
261-
reformat_script):
282+
reformat_script, start_year, end_year):
262283
"""Write the information needed by the ncl reformat script."""
284+
if start_year is None:
285+
start_year = 0
286+
else:
287+
start_year = start_year.year
288+
if end_year is None:
289+
end_year = 0
290+
else:
291+
end_year = end_year.year
263292
settings = {
264293
'cmorization_script': reformat_script,
265294
'input_dir_path': project_info[dataset]['indir'],
266295
'output_dir_path': project_info[dataset]['outdir'],
267296
'config_user_info': {
268297
'log_level': self.config['log_level'],
269298
},
299+
'start_year': start_year,
300+
'end_year': end_year,
270301
}
271302
settings_filename = os.path.join(run_dir, dataset, 'settings.ncl')
272303
if not os.path.isdir(os.path.join(run_dir, dataset)):
@@ -275,7 +306,7 @@ def _write_ncl_settings(self, project_info, dataset, run_dir,
275306
write_ncl_settings(settings, settings_filename)
276307
return settings_filename
277308

278-
def _run_ncl_script(self, in_dir, out_dir, dataset, script):
309+
def _run_ncl_script(self, in_dir, out_dir, dataset, script, start, end):
279310
"""Run the NCL cmorization mechanism."""
280311
logger.info("CMORizing dataset %s using NCL script %s",
281312
dataset, script)
@@ -284,7 +315,7 @@ def _run_ncl_script(self, in_dir, out_dir, dataset, script):
284315
project[dataset]['indir'] = in_dir
285316
project[dataset]['outdir'] = out_dir
286317
settings_file = self._write_ncl_settings(
287-
project, dataset, self.run_dir, script)
318+
project, dataset, self.run_dir, script, start, end)
288319
esmvaltool_root = os.path.dirname(
289320
os.path.dirname(os.path.dirname(os.path.dirname(script))))
290321

@@ -306,17 +337,20 @@ def _run_ncl_script(self, in_dir, out_dir, dataset, script):
306337
for oline in str(output.decode('utf-8')).split('\n'):
307338
logger.info('[NCL] %s', oline)
308339
if err:
309-
logger.info('[NCL][subprocess.Popen ERROR] %s', err)
340+
logger.error('[NCL][subprocess.Popen ERROR] %s', err)
341+
return False
342+
return True
310343

311-
def _run_pyt_script(self, in_dir, out_dir, dataset):
344+
def _run_pyt_script(self, in_dir, out_dir, dataset, start, end):
312345
"""Run the Python cmorization mechanism."""
313346
module_name = 'esmvaltool.cmorizers.obs.formatters.datasets.{}'.format(
314347
dataset.lower().replace("-", "_"))
315348
module = importlib.import_module(module_name)
316349
logger.info("CMORizing dataset %s using Python script %s",
317350
dataset, module.__file__)
318351
cmor_cfg = read_cmor_config(dataset)
319-
module.cmorization(in_dir, out_dir, cmor_cfg, self.config)
352+
module.cmorization(in_dir, out_dir, cmor_cfg, self.config, start, end)
353+
return True
320354

321355

322356
class DataCommand():
@@ -340,14 +374,18 @@ def download(self, datasets, config_file=None, start=None, end=None,
340374
formatter.start('download', datasets, config_file, kwargs)
341375
formatter.download(start, end, overwrite)
342376

343-
def format(self, datasets=None, config_file=None, **kwargs):
377+
def format(self, datasets=None, config_file=None, start=None, end=None,
378+
install=False, **kwargs):
379+
start = self._parse_date(start)
380+
end = self._parse_date(end)
381+
344382
formatter = Formatter()
345383
formatter.start('download', datasets, config_file, kwargs)
346-
formatter.format()
384+
formatter.format(start, end, install)
347385

348386
def prepare(self, datasets=None, config_file=None,
349387
start=None, end=None,
350-
overwrite=False, **kwargs):
388+
overwrite=False, install=False, **kwargs):
351389
"""
352390
Download a format a set of datasets.
353391
@@ -361,7 +399,7 @@ def prepare(self, datasets=None, config_file=None,
361399
formatter = Formatter()
362400
formatter.start('download', datasets, config_file, kwargs)
363401
formatter.download(start, end, overwrite)
364-
formatter.format()
402+
formatter.format(start, end, install)
365403

366404
@staticmethod
367405
def _parse_date(date):

‎esmvaltool/cmorizers/obs/downloaders/datasets/berkeleyearth.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
"""Script to download cds-satellite-albedo from the Climate Data Store(CDS)"""
1+
"""Script to download BerkeleyEarth from its webpage."""
22

33
from esmvaltool.cmorizers.obs.downloaders.wget import WGetDownloader
44

55

66
def download_dataset(config, dataset, start_date, end_date, overwrite):
7-
"""Download dataset cds-satellite-albedo."""
7+
"""Download dataset BerkeleyEarth."""
88
downloader = WGetDownloader(
99
config=config,
1010
dataset=dataset,

‎esmvaltool/cmorizers/obs/downloaders/datasets/calipso_goccp.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
1-
"""Script to download cds-satellite-albedo from the Climate Data Store(CDS)"""
1+
"""Script to download CALIPSO-GOCCP from IPSL ftp."""
22

33
from dateutil import relativedelta
44

55
from esmvaltool.cmorizers.obs.downloaders.ftp import FTPDownloader
66

77

88
def download_dataset(config, dataset, start_date, end_date, overwrite):
9-
"""Download dataset cds-satellite-albedo."""
9+
"""Download dataset CALIPSO-GOCCP."""
1010
downloader = FTPDownloader(
1111
config=config,
1212
server='ftp.climserv.ipsl.polytechnique.fr',

‎esmvaltool/cmorizers/obs/downloaders/datasets/esacci_aerosol.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
1-
"""Script to download cds-satellite-albedo from the Climate Data Store(CDS)"""
1+
"""Script to download ESACCI-AEROSOL from CCI CEDA ftp."""
22

33
from dateutil import relativedelta
44

55
from esmvaltool.cmorizers.obs.downloaders.ftp import CCIDownloader
66

77

88
def download_dataset(config, dataset, start_date, end_date, overwrite):
9-
"""Download dataset cds-satellite-albedo."""
9+
"""Download dataset ESACCI-AEROSOL."""
1010
loop_date = start_date
1111

1212
downloader = CCIDownloader(

‎esmvaltool/cmorizers/obs/downloaders/datasets/esacci_fire.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ def download_dataset(config, dataset, start_date, end_date, overwrite):
1111

1212
downloader = CCIDownloader(
1313
config=config,
14-
dataset=dataset
14+
dataset=dataset,
15+
overwrite=overwrite
1516
)
1617
downloader.connect()
1718

‎esmvaltool/cmorizers/obs/downloaders/datasets/esacci_l4_ghrsst_gmpe.py

+1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ def download_dataset(config, dataset, start_date, end_date, overwrite):
1717
},
1818
config=config,
1919
dataset=dataset,
20+
overwrite=overwrite,
2021
)
2122

2223
while loop_date <= end_date:

‎esmvaltool/cmorizers/obs/downloaders/datasets/esacci_oc.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ def download_dataset(config, dataset, start_date, end_date, overwrite):
1111

1212
downloader = CCIDownloader(
1313
config=config,
14-
dataset=dataset
14+
dataset=dataset,
15+
overwrite=overwrite,
1516
)
1617
downloader.ftp_name = 'ocean_colour'
1718
downloader.connect()

‎esmvaltool/cmorizers/obs/downloaders/datasets/esacci_sea_surface_salinity.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ def download_dataset(config, dataset, start_date, end_date, overwrite):
1111

1212
downloader = CCIDownloader(
1313
config=config,
14-
dataset=dataset
14+
dataset=dataset,
15+
overwrite=overwrite,
1516
)
1617
downloader.connect()
1718
downloader.set_cwd('v01.8/30days')

‎esmvaltool/cmorizers/obs/downloaders/datasets/merra2.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ def download_dataset(config, dataset, start_date, end_date, overwrite):
1111

1212
downloader = NASADownloader(
1313
config=config,
14-
dataset=dataset
14+
dataset=dataset,
15+
overwrite=overwrite,
1516
)
1617

1718
while loop_date <= end_date:

‎esmvaltool/cmorizers/obs/downloaders/wget.py

+19-20
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,14 @@ def download_folder(self, server_path, wget_options):
2222
wget_options: list(str)
2323
Extra options for wget
2424
"""
25-
subprocess.check_output(
26-
['wget'] + wget_options + self.overwrite_options + [
27-
f'--directory-prefix={self.local_folder}',
28-
'--recursive',
29-
'--no-directories',
30-
f'{server_path}',
31-
]
32-
)
25+
command = ['wget'] + wget_options + self.overwrite_options + [
26+
f'--directory-prefix={self.local_folder}',
27+
'--recursive',
28+
'--no-directories',
29+
f'{server_path}',
30+
]
31+
logger.debug(command)
32+
subprocess.check_output(command)
3333

3434
def download_file(self, server_path, wget_options):
3535
"""
@@ -42,28 +42,27 @@ def download_file(self, server_path, wget_options):
4242
wget_options: list(str)
4343
Extra options for wget
4444
"""
45-
subprocess.check_output(
46-
['wget'] + wget_options + self.overwrite_options + [
47-
f'--directory-prefix={self.local_folder}',
48-
'--no-directories',
49-
server_path,
50-
]
51-
)
45+
command = ['wget'] + wget_options + self.overwrite_options + [
46+
f'--directory-prefix={self.local_folder}',
47+
'--no-directories',
48+
server_path,
49+
]
50+
logger.debug(command)
51+
subprocess.check_output(command)
5252

5353
@property
5454
def overwrite_options(self):
5555
"""Get overwrite options as configured in downloader."""
56-
if self.overwrite:
56+
if not self.overwrite:
5757
return ['--no-clobber', ]
58-
else:
59-
[]
58+
return []
6059

6160

6261
class NASADownloader(WGetDownloader):
6362
"""Downloader for the NASA repository."""
6463

65-
def __init__(self, config, dataset):
66-
super().__init__(config, dataset)
64+
def __init__(self, config, dataset, overwrite):
65+
super().__init__(config, dataset, overwrite)
6766
self.tier = 3
6867

6968
self._wget_common_options = [

‎esmvaltool/cmorizers/obs/formatters/datasets/calipso_goccp.ncl

+2-2
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@ begin
3737
TIER = 2
3838

3939
; Period
40-
YEAR1 = 2007
41-
YEAR2 = 2015
40+
YEAR1 = get_year(start_year, 2007)
41+
YEAR2 = get_year(end_year, 2015)
4242

4343
; Selected variable (standard name)
4444
VAR = (/"clcalipso"/)

‎esmvaltool/cmorizers/obs/formatters/datasets/esacci_aerosol.ncl

+2-2
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@ begin
4242
TIER = 2
4343

4444
; Period
45-
YEAR1 = 1997
46-
YEAR2 = 2011
45+
YEAR1 = get_year(start_year, 1997)
46+
YEAR2 = get_year(end_year, 2011)
4747

4848
; Selected variable (standard name)
4949
VAR = (/"od550aer", "od870aer", "od550lt1aer", "abs550aer", \

‎esmvaltool/cmorizers/obs/formatters/datasets/esacci_cloud.ncl

+2-2
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,8 @@ begin
5050
TIER = 2
5151

5252
; Period
53-
YEAR1 = 1982
54-
YEAR2 = 2016
53+
YEAR1 = get_year(start_year, 1982)
54+
YEAR2 = get_year(end_year, 2016)
5555

5656
; Selected variable (standard name)
5757
VAR = (/"clt", "cltStderr", "clivi", "clwvi"/)

‎esmvaltool/cmorizers/obs/formatters/datasets/esacci_fire.ncl

+2-2
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,8 @@ begin
3535
TIER = 2
3636

3737
; Period
38-
YEAR1 = 2005
39-
YEAR2 = 2011
38+
YEAR1 = get_year(start_year, 2005)
39+
YEAR2 = get_year(end_year, 2011)
4040

4141
; Selected variable (standard name)
4242
VAR = "burntArea"

‎esmvaltool/cmorizers/obs/formatters/datasets/esacci_oc.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ def merge_data(in_dir, out_dir, raw_info, bins):
144144
return (datafile, dsmeta['BINNING'])
145145

146146

147-
def cmorization(in_dir, out_dir, cfg, _):
147+
def cmorization(in_dir, out_dir, cfg, _, start, end):
148148
"""Cmorization func call."""
149149
cmor_table = cfg['cmor_table']
150150
glob_attrs = cfg['attributes']

0 commit comments

Comments
 (0)