Updated CDS datasets

Author: Javier Vegas-Regidor

esmvaltool/cmorizers/obs/download_scripts/download_cds_satellite_soil_moisture.py

@@ -87,12 +87,9 @@ def download_cds_satellite_sm():
                 '11',
                 '12',
             ],
-            'day':
-            '01',
-            'type_of_record':
-            'cdr',
-            'version':
-            'v201912.0.0',
+            'day': '01',
+            'type_of_record': 'cdr',
+            'version': 'v201912.0.0',
             **freq_specific_kwargs[args.frequency]
         }
         client.retrieve('satellite-soil-moisture', request_dictionary,
@@ -135,12 +132,7 @@ def download_cds_satellite_sm():
             'version': 'v201812.0.0',
             **freq_specific_kwargs[args.frequency]
         }, savename)
-    # Unpack the file
-    tar = tarfile.open(savename)
-    tar.extractall(path=os.path.dirname(savename))
-    tar.close()
-    # Remove the tar file since it has been extracted
-    os.remove(savename)
+
 
 
 if __name__ == "__main__":

esmvaltool/cmorizers/obs/downloaders/cds.py

@@ -2,6 +2,7 @@
 
 import os
 import logging
+from collections.abc import Iterable
 import cdsapi
 
 from .downloader import BaseDownloader
@@ -12,37 +13,47 @@
 class CDSDownloader(BaseDownloader):
 
     def __init__(self, product_name, config, request_dictionary, dataset,
-                 overwrite):
+                 overwrite, extra_name=''):
         super().__init__(config, dataset, overwrite)
         self._client = cdsapi.Client()
         self._product_name = product_name
         self._request_dict = request_dictionary
+        self.extra_name = extra_name
 
     def download(self, year, month, day=None):
         request_dict = self._request_dict.copy()
         request_dict['year'] = f'{year}'
         request_dict['month'] = f"{month:02d}"
         if day:
-            request_dict['day'] = f"{day:02d}"
+            if isinstance(day, Iterable):
+                request_dict['day'] = day
+            else:
+                request_dict['day'] = f"{day:02d}"
 
-        date_str = f"{self.dataset}_{year}{month:02d}"
+        date_str = f"{self.dataset}{self.extra_name}_{year}{month:02d}"
         if day:
-            date_str += f"{day:02d}"
+            if not isinstance(day, Iterable):
+                date_str += f"{day:02d}"
 
         os.makedirs(self.local_folder, exist_ok=True)
-        file_path = os.path.join(
-            self.local_folder,
-            f"{self.dataset}_{date_str}.tar"
-        )
-        if os.path.exists(file_path):
+        file_path = f"{self.dataset}_{date_str}.tar"
+        self.download_request(file_path, request_dict)
+
+    def download_request(self, filename, request=None):
+        if request is None:
+            request = self._request_dict.copy()
+
+        os.makedirs(self.local_folder, exist_ok=True)
+        filename = os.path.join(self.local_folder, filename)
+        if os.path.exists(filename):
             if self.overwrite:
-                os.remove(file_path)
+                os.remove(filename)
             else:
                 logger.info(
-                    'File %s already downloaded. Skipping...', file_path)
+                    'File %s already downloaded. Skipping...', filename)
                 return
         self._client.retrieve(
             self._product_name,
-            request_dict,
-            file_path,
+            request,
+            filename,
         )

esmvaltool/cmorizers/obs/downloaders/datasets/aphro_ma.py

@@ -4,6 +4,7 @@
 import logging
 
 from esmvaltool.cmorizers.obs.downloaders.wget import WGetDownloader
+from esmvaltool.cmorizers.obs.utilities import unpack_files_in_folder
 
 logger = logging.getLogger(__name__)
 
@@ -13,14 +14,6 @@
 import shutil
 
 
-def gunzip(file_name, work_dir):
-    filename = os.path.split(file_name)[-1]
-    filename = re.sub(r"\.gz$", "", filename, flags=re.IGNORECASE)
-
-    with gzip.open(file_name, 'rb') as f_in:
-        with open(os.path.join(work_dir, filename), 'wb') as f_out:
-            shutil.copyfileobj(f_in, f_out)
-
 def download_dataset(config, dataset, start_date, end_date, overwrite):
     downloader = WGetDownloader(
         config=config,
@@ -41,25 +34,5 @@ def download_file(path):
     # download_file("APHRO_V1101EX_R1/APHRO_MA/050deg_nc/APHRO_MA_050deg_V1101_EXR1.nc.tgz")
 
     shutil.register_unpack_format('gz', ['.gz', ], gunzip)
-    decompress =True
-    while decompress:
-        decompress = False
-        for filename in os.listdir(downloader.local_folder):
-            full_path = os.path.join(downloader.local_folder, filename)
-            if os.path.isdir(full_path):
-                logger.info('Moving files from folder %s', filename)
-                folder_files = os.listdir(full_path)
-                for file_path in folder_files:
-                    shutil.move(
-                        os.path.join(full_path, file_path),
-                        downloader.local_folder
-                    )
-                os.rmdir(full_path)
-                decompress = True
-                continue
-            if not filename.endswith(('.gz', '.tgz', '.tar')):
-                continue
-            logger.info('Unpacking %s', filename)
-            shutil.unpack_archive(full_path, downloader.local_folder)
-            os.remove(full_path)
-            decompress = True
+
+    unpack_files_in_folder(downloader.local_folder)

esmvaltool/cmorizers/obs/downloaders/datasets/cds_satellite_albedo.py

@@ -3,6 +3,7 @@
 from dateutil import relativedelta
 
 from esmvaltool.cmorizers.obs.downloaders.cds import CDSDownloader
+from esmvaltool.cmorizers.obs.utilities import unpack_files_in_folder
 
 
 def download_dataset(config, dataset, start_date, end_date, overwrite):
@@ -31,3 +32,5 @@ def download_dataset(config, dataset, start_date, end_date, overwrite):
     while loop_date <= end_date:
         downloader.download(loop_date.year, loop_date.month)
         loop_date += relativedelta.relativedelta(months=1)
+
+    unpack_files_in_folder(downloader.local_folder)

esmvaltool/cmorizers/obs/downloaders/datasets/cds_satellite_lai_fapar.py

@@ -0,0 +1,40 @@
+"""Script to download cds-satellite-albedo from the Climate Data Store(CDS)"""
+
+from dateutil import relativedelta
+import datetime
+
+from esmvaltool.cmorizers.obs.downloaders.cds import CDSDownloader
+from esmvaltool.cmorizers.obs.utilities import unpack_files_in_folder
+
+
+def download_dataset(config, dataset, start_date, end_date, overwrite):
+    """Download dataset cds-satellite-albedo."""
+    if not start_date:
+        start_date = datetime.datetime(1998, 4, 1)
+    if not end_date:
+        end_date = datetime.datetime(2014, 5, 1)
+
+    loop_date = start_date
+    downloader = CDSDownloader(
+        product_name='satellite-lai-fapar',
+        request_dictionary={
+            'variable': [
+                'fapar', 'lai',
+            ],
+            'satellite': 'spot',
+            'sensor': 'vgt',
+            'horizontal_resolution': '1km',
+            'product_version': 'V1',
+            'nominal_day': '20',
+            'format': 'tgz',
+        },
+        config=config,
+        dataset=dataset,
+        overwrite=overwrite,
+    )
+
+    while loop_date <= end_date:
+        downloader.download(loop_date.year, loop_date.month)
+        loop_date += relativedelta.relativedelta(months=1)
+
+    unpack_files_in_folder(downloader.local_folder)

esmvaltool/cmorizers/obs/downloaders/datasets/cds_satellite_soil_moisture.py

@@ -0,0 +1,63 @@
+"""Script to download cds-satellite-soil-moisture from the Climate Data Store(CDS)"""
+
+from dateutil import relativedelta
+import datetime
+import calendar
+
+from esmvaltool.cmorizers.obs.downloaders.cds import CDSDownloader
+from esmvaltool.cmorizers.obs.utilities import unpack_files_in_folder
+
+
+def download_dataset(config, dataset, start_date, end_date, overwrite):
+    """Download dataset cds-satellite-soil-moisture."""
+    if not start_date:
+        start_date = datetime.datetime(1978, 11, 1)
+    if not end_date:
+        end_date = datetime.datetime(2020, 6, 30)
+
+    # loop_date = start_date
+    # downloader = CDSDownloader(
+    #     product_name='satellite-soil-moisture',
+    #     request_dictionary={
+    #         'format': 'tgz',
+    #         'variable': 'volumetric_surface_soil_moisture',
+    #         'type_of_sensor': 'combined_passive_and_active',
+    #         'type_of_record': 'cdr',
+    #         'version': 'v201912.0.0',
+    #         'time_aggregation': 'month_average',
+    #         'day': ['01']
+    #     },
+    #     config=config,
+    #     dataset=dataset,
+    #     overwrite=overwrite,
+    # )
+
+    # while loop_date <= end_date:
+    #     downloader.download(loop_date.year, loop_date.month)
+    #     loop_date += relativedelta.relativedelta(months=1)
+
+    downloader = CDSDownloader(
+        product_name='satellite-soil-moisture',
+        request_dictionary={
+            'format': 'tgz',
+            'variable': 'volumetric_surface_soil_moisture',
+            'type_of_sensor': 'combined_passive_and_active',
+            'day': '01',
+            'type_of_record': 'cdr',
+            'version': 'v201912.0.0',
+            'time_aggregation': 'day_average',
+        },
+        config=config,
+        dataset=dataset,
+        overwrite=overwrite,
+        extra_name='_monthly',
+    )
+    loop_date = start_date
+    while loop_date <= end_date:
+        downloader.download(
+            loop_date.year, loop_date.month,
+            [f'{i+1:02d}' for i in range(calendar.monthrange(loop_date.year, loop_date.month)[1])])
+        loop_date += relativedelta.relativedelta(months=1)
+
+    unpack_files_in_folder(downloader.local_folder)
+

esmvaltool/cmorizers/obs/downloaders/datasets/cds_uerra.py

@@ -0,0 +1,40 @@
+"""Script to download cds-satellite-albedo from the Climate Data Store(CDS)"""
+
+from dateutil import relativedelta
+import datetime
+import calendar
+
+from esmvaltool.cmorizers.obs.downloaders.cds import CDSDownloader
+from esmvaltool.cmorizers.obs.utilities import unpack_files_in_folder
+
+
+def download_dataset(config, dataset, start_date, end_date, overwrite):
+    """Download dataset cds-satellite-albedo."""
+    if not start_date:
+        start_date = datetime.datetime(1961, 1, 1)
+    if not end_date:
+        end_date = datetime.datetime(2019, 7, 1)
+
+    loop_date = start_date
+    downloader = CDSDownloader(
+        product_name='reanalysis-uerra-europe-soil-levels',
+        request_dictionary={
+            'format': 'netcdf',
+            'origin': 'uerra_harmonie',
+            'variable': 'volumetric_soil_moisture',
+            'soil_level': ['1', '2', '3',],
+            'time': ['00:00', '06:00', '12:00', '18:00'],
+        },
+        config=config,
+        dataset=dataset,
+        overwrite=overwrite,
+    )
+
+    loop_date = start_date
+    while loop_date <= end_date:
+        downloader.download(
+            loop_date.year, loop_date.month,
+            [f'{i+1:02d}' for i in range(calendar.monthrange(loop_date.year, loop_date.month)[1])])
+        loop_date += relativedelta.relativedelta(months=1)
+
+    unpack_files_in_folder(downloader.local_folder)

esmvaltool/cmorizers/obs/downloaders/datasets/cds_xch4.py

@@ -0,0 +1,27 @@
+"""Script to download cds-satellite-albedo from the Climate Data Store(CDS)"""
+
+from dateutil import relativedelta
+import datetime
+import calendar
+
+from esmvaltool.cmorizers.obs.downloaders.cds import CDSDownloader
+from esmvaltool.cmorizers.obs.utilities import unpack_files_in_folder
+
+
+def download_dataset(config, dataset, start_date, end_date, overwrite):
+    """Download dataset cds-satellite-albedo."""
+    downloader = CDSDownloader(
+        product_name='satellite-methane',
+        request_dictionary={
+            'format': 'tgz',
+            'processing_level': 'level_3',
+            'variable': 'xch4',
+            'sensor_and_algorithm': 'merged_obs4mips',
+            'version': '4.1',
+        },
+        config=config,
+        dataset=dataset,
+        overwrite=overwrite,
+    )
+    downloader.download_request("CDS-XCH4.tar")
+    unpack_files_in_folder(downloader.local_folder)

esmvaltool/cmorizers/obs/formatters/datasets/aura_tes.ncl

@@ -44,8 +44,8 @@ begin
   GRIDTYPE = "Nadir"
 
   ; Period
-  YEAR1 = 2005
-  YEAR2 = 2011
+  YEAR1 = get_year(start_year, 2005)
+  YEAR2 = get_year(end_year, 2011)
 
   ; Selected variable
   VAR = "tro3"

esmvaltool/cmorizers/obs/formatters/datasets/cds_satellite_albedo.py

@@ -163,7 +163,7 @@ def _set_time_bnds(in_dir, var):
     return cubelist
 
 
-def cmorization(in_dir, out_dir, cfg, cfg_user):
+def cmorization(in_dir, out_dir, cfg, cfg_user, start, end):
     """Cmorization func call."""
     # run the cmorization
     # Pass on the workdir to the cfg dictionary

esmvaltool/cmorizers/obs/formatters/datasets/cds_satellite_lai_fapar.py

@@ -170,7 +170,7 @@ def _set_time_bnds(in_dir, var):
     return cubelist
 
 
-def cmorization(in_dir, out_dir, cfg, cfg_user):
+def cmorization(in_dir, out_dir, cfg, cfg_user, start, end):
     """Cmorization func call."""
     # run the cmorization
     # Pass on the workdir to the cfg dictionary

esmvaltool/cmorizers/obs/formatters/datasets/cds_satellite_soil_moisture.ncl

@@ -34,8 +34,8 @@ begin
   TIER = 3
 
   ; Period
-  YEAR1 = 1978
-  YEAR2 = 2020
+  YEAR1 = get_year(start_year, 1978)
+  YEAR2 = get_year(end_year, 2020)
 
   ; Selected variables (standard name)
   VAR = (/"sm", "sm"/)