Fix units after derivation (#754)

* Added check for units at the end of derivation function

* Fixed tests for variable derivation interface

* Adapted several derivation scripts

* Adapted sm derivation script for CMIP6

* Fixed derivation of vegfrac and ohc

* Fixed CMOR checker when coordinate standard_name is empty

Author: schlunma

doc/changelog.rst

@@ -12,8 +12,9 @@ Bug fixes
 -  Fix diagnostic filter (`#713 <https://github.com/ESMValGroup/ESMValCore/pull/713>`__) `Javier Vegas-Regidor <https://github.com/jvegasbsc>`__
 -  Set unit=1 if anomalies are standardized (`#727 <https://github.com/ESMValGroup/ESMValCore/pull/727>`__) `bascrezee <https://github.com/bascrezee>`__
 -  Fix crash for FGOALS-g2 variables without longitude coordinate (`#729 <https://github.com/ESMValGroup/ESMValCore/pull/729>`__) `Bouwe Andela <https://github.com/bouweandela>`__
--  Improve variable alias managament (`#595 <https://github.com/ESMValGroup/ESMValCore/pull/595>`__) `Javier Vegas-Regidor <https://github.com/jvegasbsc>`__
+-  Improve variable alias management (`#595 <https://github.com/ESMValGroup/ESMValCore/pull/595>`__) `Javier Vegas-Regidor <https://github.com/jvegasbsc>`__
 -  Fix area_statistics fx files loading (`#798 <https://github.com/ESMValGroup/ESMValCore/pull/798>`__) `Javier Vegas-Regidor <https://github.com/jvegasbsc>`__
+-  Fix units after derivation (`#754 <https://github.com/ESMValGroup/ESMValCore/pull/754>`__) `Manuel Schlund <https://github.com/schlunma>`__
 
 Documentation
 ~~~~~~~~~~~~~

esmvalcore/cmor/check.py

@@ -338,7 +338,10 @@ def _check_dim_names(self):
             else:
                 try:
                     cube_coord = self._cube.coord(var_name=coordinate.out_name)
-                    if cube_coord.standard_name != coordinate.standard_name:
+                    if (cube_coord.standard_name is None and
+                            coordinate.standard_name == ''):
+                        pass
+                    elif cube_coord.standard_name != coordinate.standard_name:
                         self.report_critical(
                             self._attr_msg,
                             coordinate.out_name,

esmvalcore/preprocessor/_derive/__init__.py

@@ -103,9 +103,24 @@ def derive(cubes, short_name, long_name, units, standard_name=None):
     cube.var_name = short_name
     cube.standard_name = standard_name if standard_name else None
     cube.long_name = long_name
-    cube.units = units
     for temp in cubes:
         if 'source_file' in temp.attributes:
             cube.attributes['source_file'] = temp.attributes['source_file']
 
+    # Check/convert units
+    if cube.units is None or cube.units == units:
+        cube.units = units
+    elif cube.units.is_no_unit() or cube.units.is_unknown():
+        logger.warning(
+            "Units of cube after executing derivation script of '%s' are "
+            "'%s', automatically setting them to '%s'. This might lead to "
+            "incorrect data", short_name, cube.units, units)
+        cube.units = units
+    elif cube.units.is_convertible(units):
+        cube.convert_units(units)
+    else:
+        raise ValueError(
+            f"Units '{cube.units}' after executing derivation script of "
+            f"'{short_name}' cannot be converted to target units '{units}'")
+
     return cube

esmvalcore/preprocessor/_derive/_shared.py

@@ -4,13 +4,14 @@
 
 import iris
 
+from esmvalcore.iris_helpers import var_name_constraint
+
 logger = logging.getLogger(__name__)
 
 
 def cloud_area_fraction(cubes, tau_constraint, plev_constraint):
     """Calculate cloud area fraction for different parameters."""
-    clisccp_cube = cubes.extract_strict(
-        iris.Constraint(name='isccp_cloud_area_fraction'))
+    clisccp_cube = cubes.extract_strict(var_name_constraint('clisccp'))
     new_cube = clisccp_cube
     new_cube = new_cube.extract(tau_constraint & plev_constraint)
     coord_names = [

esmvalcore/preprocessor/_derive/lvp.py

@@ -4,7 +4,8 @@
     - weig_ka
 
 """
-from iris import Constraint
+
+from esmvalcore.iris_helpers import var_name_constraint
 
 from ._baseclass import DerivedVariableBase
 
@@ -31,11 +32,9 @@ def required(project):
     @staticmethod
     def calculate(cubes):
         """Compute Latent Heat Release from Precipitation."""
-        hfls_cube = cubes.extract_strict(
-            Constraint(name='surface_upward_latent_heat_flux'))
-        pr_cube = cubes.extract_strict(Constraint(name='precipitation_flux'))
-        evspsbl_cube = cubes.extract_strict(
-            Constraint(name='water_evaporation_flux'))
+        hfls_cube = cubes.extract_strict(var_name_constraint('hfls'))
+        pr_cube = cubes.extract_strict(var_name_constraint('pr'))
+        evspsbl_cube = cubes.extract_strict(var_name_constraint('evspsbl'))
 
         lvp_cube = hfls_cube * (pr_cube / evspsbl_cube)
 

esmvalcore/preprocessor/_derive/ohc.py

@@ -62,6 +62,7 @@ def calculate(cubes):
         # 2. multiply with each other and with cprho0
         # some juggling with coordinates needed since Iris is very
         # restrictive in this regard
+        cube.convert_units('K')
         try:
             t_coord_dims = cube.coord_dims('time')
         except iris.exceptions.CoordinateNotFoundError:

esmvalcore/preprocessor/_derive/sispeed.py

@@ -16,11 +16,10 @@ class DerivedVariable(DerivedVariableBase):
     @staticmethod
     def required(project):
         """Declare the variables needed for derivation."""
-        required = [{
-            'short_name': 'usi',
-        }, {
-            'short_name': 'vsi',
-        }]
+        if project == 'CMIP6':
+            required = [{'short_name': 'siu'}, {'short_name': 'siv'}]
+        else:
+            required = [{'short_name': 'usi'}, {'short_name': 'vsi'}]
         return required
 
     @staticmethod

esmvalcore/preprocessor/_derive/sm.py

@@ -2,7 +2,8 @@
 
 import cf_units
 import numpy as np
-from iris import Constraint
+
+from esmvalcore.iris_helpers import var_name_constraint
 
 from ._baseclass import DerivedVariableBase
 
@@ -27,8 +28,7 @@ def calculate(cubes):
         20 deg C).
 
         """
-        mrsos_cube = cubes.extract_strict(
-            Constraint(name='moisture_content_of_soil_layer'))
+        mrsos_cube = cubes.extract_strict(var_name_constraint('mrsos'))
 
         depth = mrsos_cube.coord('depth').bounds.astype(np.float32)
         layer_thickness = depth[..., 1] - depth[..., 0]

esmvalcore/preprocessor/_derive/toz.py

@@ -26,14 +26,10 @@ class DerivedVariable(DerivedVariableBase):
     @staticmethod
     def required(project):
         """Declare the variables needed for derivation."""
-        required = [
-            {
-                'short_name': 'tro3'
-            },
-            {
-                'short_name': 'ps'
-            },
-        ]
+        if project == 'CMIP6':
+            required = [{'short_name': 'o3'}, {'short_name': 'ps'}]
+        else:
+            required = [{'short_name': 'tro3'}, {'short_name': 'ps'}]
         return required
 
     @staticmethod
@@ -64,6 +60,7 @@ def calculate(cubes):
         toz_cube = toz_cube / MW_O3 * AVOGADRO_CONST
         toz_cube.units = toz_cube.units / MW_O3_UNIT * AVOGADRO_CONST_UNIT
         toz_cube.convert_units(DOBSON_UNIT)
+        toz_cube.units = 'DU'
 
         return toz_cube
 

esmvalcore/preprocessor/_derive/vegfrac.py

@@ -1,6 +1,11 @@
 """Derivation of variable `vegFrac`."""
 
+import dask.array as da
 import iris
+from esmvalcore.iris_helpers import var_name_constraint
+
+from .._regrid import regrid
+
 from ._baseclass import DerivedVariableBase
 
 
@@ -10,16 +15,38 @@ class DerivedVariable(DerivedVariableBase):
     @staticmethod
     def required(project):
         """Declare the variables needed for derivation."""
-        required = [{
-            'short_name': 'baresoilFrac',
-        }]
+        required = [
+            {'short_name': 'baresoilFrac'},
+            {'short_name': 'residualFrac'},
+            {'short_name': 'sftlf', 'mip': 'fx'},
+        ]
         return required
 
     @staticmethod
     def calculate(cubes):
         """Compute vegetation fraction from bare soil fraction."""
-        baresoilfrac_cube = cubes.extract_strict(
-            iris.Constraint(name='area_fraction'))
+        baresoilfrac_cube = cubes.extract_strict(var_name_constraint(
+            'baresoilFrac'))
+        residualfrac_cube = cubes.extract_strict(var_name_constraint(
+            'residualFrac'))
+        sftlf_cube = cubes.extract_strict(var_name_constraint('sftlf'))
+
+        # Add time dimension to sftlf
+        target_shape_sftlf = (baresoilfrac_cube.shape[0], *sftlf_cube.shape)
+        sftlf_data = iris.util.broadcast_to_shape(sftlf_cube.data,
+                                                  target_shape_sftlf, (1, 2))
+        sftlf_cube = baresoilfrac_cube.copy(sftlf_data)
+        sftlf_cube.data = sftlf_cube.lazy_data()
+
+        # Regrid sftlf if necessary and adapt mask
+        if sftlf_cube.shape != baresoilfrac_cube.shape:
+            sftlf_cube = regrid(sftlf_cube, baresoilfrac_cube, 'linear')
+        sftlf_cube.data = da.ma.masked_array(
+            sftlf_cube.core_data(),
+            mask=da.ma.getmaskarray(baresoilfrac_cube.core_data()))
 
-        baresoilfrac_cube.data = 1. - baresoilfrac_cube.core_data()
+        # Calculate vegetation fraction
+        baresoilfrac_cube.data = (sftlf_cube.core_data() -
+                                  baresoilfrac_cube.core_data() -
+                                  residualfrac_cube.core_data())
         return baresoilfrac_cube