Merge remote-tracking branch 'upstream/master' into im1787

* upstream/master:
  Fix to the wind vertical displacement adjustment implementation (metoppv#1927)
  Add function which normalises input cubes according to a reference (metoppv#1919)
  Skip ECC bounds usage when converting probabilities to percentiles (metoppv#1926)
  Add CLIs to support rescaling of the forecast based on altitude difference (metoppv#1917)
  Changes to the modal code to increase the percentage to 30% and change the groupings to provide a more representative daily summary symbol. (metoppv#1925)
  Add plugins to support rescaling of the forecast based on altitude difference (metoppv#1916)
  Support conversion from percentiles to probabilities (metoppv#1924)
  Correct handling of reference time in weather_code plugin (metoppv#1920)
  Add CLI for clipping cubes (metoppv#1918)
  Update cbh ecc name (metoppv#1922)
  Updates Broadcast and expand_bounds in Combine Plugin (metoppv#1914)
  Mobt515 cloud base height spot extraction (metoppv#1911)
  MOBT-494: Cube title setting in weather symbol code (metoppv#1912)
  MOBT512-masking percentiles for cloud base height (metoppv#1908)
  Mobt 496 enforce forecast between references (metoppv#1907)

improver/calibration/utilities.py

@@ -166,12 +166,21 @@ def check_predictor(predictor: str) -> str:
 
 
 def filter_non_matching_cubes(
-    historic_forecast: Cube, truth: Cube
+    historic_forecast: Cube, truth: Cube,
 ) -> Tuple[Cube, Cube]:
     """
     Provide filtering for the historic forecast and truth to make sure
     that these contain matching validity times. This ensures that any
     mismatch between the historic forecasts and truth is dealt with.
+    If multiple time slices of the historic forecast match with the
+    same truth slice, only the first truth slice is kept to avoid
+    duplicate truth slices, which prevent the truth cubes being merged.
+    This can occur when processing a cube with a multi-dimensional time
+    coordinate. If a historic forecast time slice contains only NaNs,
+    then this time slice is also skipped. This can occur when processing
+    a multi-dimensional time coordinate where some of the forecast reference
+    time and forecast period combinations do not typically occur, so may
+    be filled with NaNs.
 
     Args:
         historic_forecast:
@@ -193,6 +202,7 @@ def filter_non_matching_cubes(
     """
     matching_historic_forecasts = iris.cube.CubeList([])
     matching_truths = iris.cube.CubeList([])
+    truth_times = []
     for hf_slice in historic_forecast.slices_over("time"):
         if hf_slice.coord("time").has_bounds():
             point = iris_time_to_datetime(
@@ -216,7 +226,12 @@ def filter_non_matching_cubes(
         constr = iris.Constraint(coord_values=coord_values)
         truth_slice = truth.extract(constr)
 
-        if truth_slice:
+        if (
+            truth_slice
+            and not np.isnan(hf_slice.data).all()
+            and truth_slice.coord("time").cell(0) not in truth_times
+        ):
+            truth_times.append(truth_slice.coord("time").cell(0))
             matching_historic_forecasts.append(hf_slice)
             matching_truths.append(truth_slice)
     if not matching_historic_forecasts and not matching_truths:

improver/cli/enforce_consistent_probabilities.py → improver/cli/apply_dz_rescaling.py

@@ -29,61 +29,44 @@
 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
-"""CLI to enforce consistent probabilities between two forecasts."""
+"""CLI to apply a scaling factor to account for a correction linked to the
+difference in altitude between the grid point and the site location."""
 
 from improver import cli
 
 
 @cli.clizefy
 @cli.with_output
 def process(
-    *cubes: cli.inputcubelist, ref_name: str = None, diff_for_warning: float = None
+    forecast: cli.inputcube,
+    scaling_factor: cli.inputcube,
+    *,
+    site_id_coord: str = "wmo_id",
 ):
-    """Module to enforce consistent probabilities between two forecast
-    cubes by lowering the probabilities in the forecast cube to be less than or
-    equal to the reference forecast.
+    """Apply a scaling factor to account for a correction linked to the difference
+    in altitude between the grid point and the site location.
 
     Args:
-        cubes (iris.cube.CubeList or list of iris.cube.Cube):
-            containing:
-                forecast_cube (iris.cube.Cube):
-                    Cube of probabilities
-                ref_forecast (iris.cube.Cube)
-                    Cube of probabilities used as the upper cap for
-                    forecast_cube probabilities. It must be the same shape as
-                    forecast_cube but have a different name.
-
-        ref_name (str):
-            Name of ref_forecast cube
-        diff_for_warning (float):
-            A float between 0 and 1. If assigned, the plugin will raise a warning
-            if the forecast probabilities are decreased by more than this value.
+        forecast (iris.cube.Cube):
+            Percentile forecasts.
+        rescaling_cube (iris.cube.Cube):
+            Multiplicative scaling factor to adjust the percentile forecasts.
+            This cube is expected to contain multiple values for the forecast_period
+            and forecast_reference_time_hour coordinates. The most appropriate
+            forecast period and forecast reference_time_hour pair within the
+            rescaling cube are chosen using the forecast reference time hour from
+            the forecast and the nearest forecast period that is greater than or
+            equal to the forecast period of the forecast. This cube is generated
+            using the estimate_dz_rescaling CLI.
+        site_id_coord (str):
+            The name of the site ID coordinate. This defaults to 'wmo_id'.
 
     Returns:
         iris.cube.Cube:
-            Cube with identical metadata to forecast_cube but with
-            lowered probabilities to be less than or equal to the
-            reference forecast
+            Percentile forecasts that have been rescaled to account for a difference
+            in altitude between the grid point and the site location.
     """
-    from iris.cube import CubeList
-
-    from improver.calibration.reliability_calibration import (
-        EnforceConsistentProbabilities,
-    )
-    from improver.utilities.flatten import flatten
-
-    cubes = flatten(cubes)
-
-    if len(cubes) != 2:
-        raise ValueError(
-            f"Exactly two cubes should be provided but received {len(cubes)}"
-        )
-
-    ref_forecast = CubeList(cubes).extract_cube(ref_name)
-    cubes.remove(ref_forecast)
-
-    forecast_cube = cubes[0]
 
-    plugin = EnforceConsistentProbabilities(diff_for_warning=diff_for_warning)
+    from improver.calibration.dz_rescaling import ApplyDzRescaling
 
-    return plugin(forecast_cube, ref_forecast)
+    return ApplyDzRescaling(site_id_coord=site_id_coord)(forecast, scaling_factor)

improver/cli/apply_height_adjustment.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# -----------------------------------------------------------------------------
+# (C) British Crown copyright. The Met Office.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+"""Script to apply height adjustments to spot data."""
+
+from improver import cli
+
+
+@cli.clizefy
+@cli.with_output
+def process(
+    spot_cube: cli.inputcube,
+    neighbour: cli.inputcube,
+    *,
+    land_constraint: bool = False,
+    similar_altitude: bool = False,
+):
+    """Apply height adjustment to account for the difference between site altitude and
+    grid square orography. The spot forecast contains information representative of the
+    associated grid point. This needs to be adjusted to reflect the true site altitude.
+
+    Args:
+        spot_cube (iris.cube.Cube):
+            A cube of spot forecasts. If this is a cube of probabilities
+            then the units of the threshold coordinate must be convertible to
+            metres as this is expected to represent a vertical coordinate.
+            If this is a cube of percentiles or realizations then the
+            units of the cube must be convertible to metres as the cube is
+            expected to represent a vertical profile.
+        neighbour (iris.cube.Cube):
+            A cube containing information about spot-data neighbours and
+            the spot site information.
+        land_constraint (bool):
+            Use to select the nearest-with-land-constraint neighbour-selection
+            method from the neighbour_cube. This means that the grid points
+            should be land points except for sites where none were found within
+            the search radius when the neighbour cube was created. May be used
+            with similar_altitude.
+        similar_altitude (bool):
+            Use to select the nearest-with-height-constraint
+            neighbour-selection method from the neighbour_cube. These are grid
+            points that were found to be the closest in altitude to the spot
+            site within the search radius defined when the neighbour cube was
+            created. May be used with land_constraint.
+
+    Returns:
+        iris.cube.Cube:
+            A cube of spot data values with the same metadata as spot_cube but with data
+            adjusted to be relative to site height rather than orography grid square
+            height
+    """
+    from improver.spotdata.height_adjustment import SpotHeightAdjustment
+    from improver.spotdata.neighbour_finding import NeighbourSelection
+
+    neighbour_selection_method = NeighbourSelection(
+        land_constraint=land_constraint, minimum_dz=similar_altitude
+    ).neighbour_finding_method_name()
+
+    result = SpotHeightAdjustment(neighbour_selection_method)(spot_cube, neighbour)
+    return result

improver/cli/clip.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# -----------------------------------------------------------------------------
+# (C) British Crown copyright. The Met Office.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+"""Script to clip the input cube's data to be between the specified values"""
+
+from improver import cli
+
+
+@cli.clizefy
+@cli.with_output
+def process(
+    cube: cli.inputcube, *, min_value: float = None, max_value: float = None,
+):
+    """Clip the data in the input cube such that any data above max_value is set equal to
+     max_value and any data below min_value is set equal to min_value.
+
+    Args:
+        cube (iris.cube.Cube):
+            A Cube whose data will be clipped. This can be a cube of spot or gridded data.
+        max_value (float):
+            If specified any data in cube that is above max_value will be set equal to
+            max_value.
+        min_value (float):
+            If specified any data in cube that is below min_value will be set equal to
+            min_value.
+    Returns:
+        iris.cube.Cube:
+            A cube with the same metadata as the input cube but with the data clipped such
+            that any data above max_value is set equal to max_value and any data below
+            min_value is set equal to min_value.
+    """
+    from numpy import clip
+
+    cube.data = clip(cube.data, min_value, max_value)
+
+    return cube

improver/cli/combine.py

@@ -40,9 +40,10 @@ def process(
     *cubes: cli.inputcube,
     operation="+",
     new_name=None,
-    broadcast_to_threshold=False,
+    broadcast=None,
     minimum_realizations=None,
     cell_method_coordinate=None,
+    expand_bound=True,
 ):
     r"""Combine input cubes.
 
@@ -60,17 +61,19 @@ def process(
             +, -, \*, add, subtract, multiply, min, max, mean
         new_name (str):
             New name for the resulting dataset.
-        broadcast_to_threshold (bool):
-            If True, broadcast input cubes to the threshold coord prior to combining -
-            a threshold coord must already exist on the first input cube.
+        broadcast (str):
+            If specified, the input cubes will be broadcast over the coordinate name provided. If
+            "threshold" is provided the plugin will try to find a threshold coordinate on the
+            probability cube.
         minimum_realizations (int):
             If specified, the input cubes will be filtered to ensure that only realizations that
             include all available lead times are combined. If the number of realizations that
             meet this criteria are fewer than this integer, an error will be raised.
         cell_method_coordinate (str):
             If specified, a cell method is added to the output with the coordinate
             provided. This is only available for max, min and mean operations.
-
+        expand_bound (bool):
+            If True then coord bounds will be extended to represent all cubes being combined.
     Returns:
         result (iris.cube.Cube):
             Returns a cube with the combined data.
@@ -81,8 +84,9 @@ def process(
 
     return Combine(
         operation,
-        broadcast_to_threshold=broadcast_to_threshold,
+        broadcast=broadcast,
         minimum_realizations=minimum_realizations,
         new_name=new_name,
         cell_method_coordinate=cell_method_coordinate,
+        expand_bound=expand_bound,
     )(CubeList(cubes))