Fix issue with reading dates from files (#1936)

Co-authored-by: Rémi Kazeroni <70641264+remi-kazeroni@users.noreply.github.com>

Author: bouweandela

esmvalcore/_recipe/check.py

@@ -172,7 +172,7 @@ def data_availability(dataset, log=True):
     available_years = set()
 
     for file in input_files:
-        start, end = _get_start_end_year(file.name)
+        start, end = _get_start_end_year(file)
         available_years.update(range(start, end + 1))
 
     missing_years = required_years - available_years

esmvalcore/local.py

@@ -162,7 +162,7 @@ def _replace_years_with_timerange(variable):
     variable.pop('end_year', None)
 
 
-def _get_start_end_year(filename):
+def _get_start_end_year(file):
     """Get the start and end year from a file name.
 
     Examples of allowed dates : 1980, 198001, 19801231,
@@ -174,26 +174,42 @@ def _get_start_end_year(filename):
 
     Look first for two dates separated by - or _, then for one single
     date, and if they are multiple, for one date at start or end.
+
+    Parameters
+    ----------
+    file: LocalFile or esmvalcore.esgf.ESGFFile
+        The file to read the start and end year from.
+
+    Returns
+    -------
+    tuple[int, int]
+        The start and end year.
+
+    Raises
+    ------
+    ValueError
+        When start or end year cannot be determined.
+
     """
-    stem = Path(filename).stem
     start_year = end_year = None
-    #
+
     time_pattern = (r"(?P<hour>[0-2][0-9]"
                     r"(?P<minute>[0-5][0-9]"
                     r"(?P<second>[0-5][0-9])?)?Z?)")
     date_pattern = (r"(?P<year>[0-9]{4})"
                     r"(?P<month>[01][0-9]"
                     r"(?P<day>[0-3][0-9]"
                     rf"(T?{time_pattern})?)?)?")
-    #
+
     end_date_pattern = date_pattern.replace(">", "_end>")
     date_range_pattern = date_pattern + r"[-_]" + end_date_pattern
     start_year, end_year = _get_from_pattern(date_pattern, date_range_pattern,
-                                             stem, 'year')
+                                             Path(file.name).stem, 'year')
     # As final resort, try to get the dates from the file contents
-    if (start_year is None or end_year is None) and Path(filename).exists():
-        logger.debug("Must load file %s for daterange ", filename)
-        cubes = iris.load(filename)
+    if ((start_year is None or end_year is None) and isinstance(file, Path)
+            and file.exists()):
+        logger.debug("Must load file %s for daterange ", file)
+        cubes = iris.load(file)
 
         for cube in cubes:
             logger.debug(cube)
@@ -206,7 +222,7 @@ def _get_start_end_year(filename):
             break
 
     if start_year is None or end_year is None:
-        raise ValueError(f'File {filename} dates do not match a recognized '
+        raise ValueError(f'File {file} dates do not match a recognized '
                          'pattern and time can not be read from the file')
 
     return int(start_year), int(end_year)

tests/unit/local/test_time.py

@@ -3,6 +3,7 @@
 import pytest
 
 from esmvalcore.local import (
+    LocalFile,
     _dates_to_timerange,
     _get_start_end_date,
     _get_start_end_year,
@@ -64,6 +65,7 @@
 def test_get_start_end_year(case):
     """Tests for _get_start_end_year function."""
     filename, case_start, case_end = case
+    filename = LocalFile(filename)
     if case_start is None and case_end is None:
         # If the filename is inconclusive or too difficult
         # we resort to reading the file, which fails here
@@ -80,6 +82,7 @@ def test_get_start_end_year(case):
 def test_get_start_end_date(case):
     """Tests for _get_start_end_date function."""
     filename, case_start, case_end = case
+    filename = LocalFile(filename)
     if case_start is None and case_end is None:
         # If the filename is inconclusive or too difficult
         # we resort to reading the file, which fails here
@@ -95,7 +98,7 @@ def test_get_start_end_date(case):
 def test_read_time_from_cube(monkeypatch, tmp_path):
     """Try to get time from cube if no date in filename."""
     monkeypatch.chdir(tmp_path)
-    temp_file = 'test.nc'
+    temp_file = LocalFile('test.nc')
     cube = iris.cube.Cube([0, 0], var_name='var')
     time = iris.coords.DimCoord([0, 366],
                                 'time',
@@ -142,9 +145,7 @@ def test_fails_if_no_date_present():
 def test_get_timerange_from_years():
     """Test a `timerange` tag with value `start_year/end_year` can be built
     from tags `start_year` and `end_year`."""
-    variable = {
-        'start_year': 2000,
-        'end_year': 2002}
+    variable = {'start_year': 2000, 'end_year': 2002}
 
     _replace_years_with_timerange(variable)
 
@@ -156,9 +157,7 @@ def test_get_timerange_from_years():
 def test_get_timerange_from_start_year():
     """Test a `timerange` tag with value `start_year/start_year` can be built
     from tag `start_year` when an `end_year` is not given."""
-    variable = {
-        'start_year': 2000
-    }
+    variable = {'start_year': 2000}
 
     _replace_years_with_timerange(variable)
 
@@ -169,9 +168,7 @@ def test_get_timerange_from_start_year():
 def test_get_timerange_from_end_year():
     """Test a `timerange` tag with value `end_year/end_year` can be built from
     tag `end_year` when a `start_year` is not given."""
-    variable = {
-        'end_year': 2002
-    }
+    variable = {'end_year': 2002}
 
     _replace_years_with_timerange(variable)