From 3fff0949968b7fbdc3b6a965668dc79a075c8d9b Mon Sep 17 00:00:00 2001 From: AS Date: Fri, 26 Feb 2021 10:35:28 +0100 Subject: [PATCH 01/39] fix rps formula, rm clip, rm limit tests, allow many category_edges --- xskillscore/core/probabilistic.py | 123 ++++++++++++++++++------ xskillscore/tests/test_probabilistic.py | 9 -- 2 files changed, 93 insertions(+), 39 deletions(-) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index 68f4850f..4d0e6f61 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -488,26 +488,34 @@ def rps( """Calculate Ranked Probability Score. .. math:: - RPS(p, k) = 1/M \\sum_{m=1}^{M} + RPS(p, k) = \\sum_{m=1}^{M} [(\\sum_{k=1}^{m} p_k) - (\\sum_{k=1}^{m} o_k)]^{2} Parameters ---------- observations : xarray.Dataset or xarray.DataArray The observations or set of observations of the event. + Further requirements are specified based on ``category_edges``. Data should be boolean or logical \ (True or 1 for event occurance, False or 0 for non-occurance). forecasts : xarray.Dataset or xarray.DataArray The forecast likelihoods of the event. + Further requirements are specified based on ``category_edges``. + category_edges : array_like, xr.Dataset, xr.DataArray, None + - array_like: Category bin edges used to compute the CDFs based on boolean or logical + (True or 1 for event occurance, False or 0 for non-occurance) observations. If ``fair==False``, forecasts should be between 0 and 1 without a dimension ``member_dim`` or should be boolean (True,False) or binary (0, 1) containing a member dimension (probabilities will be internally calculated by ``.mean(member_dim))``. If ``fair==True``, forecasts must be boolean (True,False) or binary (0, 1) containing dimension ``member_dim``. - category_edges : array_like - Category bin edges used to compute the CDFs. Similar to np.histogram, \ - all but the last (righthand-most) bin include the left edge and exclude \ - the right edge. The last bin includes both edges. + Similar to np.histogram, all but the last (righthand-most) bin include the left edge + and exclude the right edge. The last bin includes both edges. + - xr.Dataset/xr.DataArray: edges of the categories in absolute units provided as dimension + ``category_dim``, ``threshold`` or ``quantile``. Forecasts and Observations are also in absolute units. + - tuple of xr.Dataset/xr.DataArray: same as xr.Dataset/xr.DataArray where the first item + is taken as category_edges for observations and the second item for category_edges for forecasts + - None: expect than observations and forecasts are already CDFs containing ``category_dim`` dimension dim : str or list of str, optional Dimension over which to compute mean after computing ``rps``. Defaults to None implying averaging over all dimensions. @@ -550,48 +558,103 @@ def rps( * C. A. T. Ferro. Fair scores for ensemble forecasts. Q.R.J. Meteorol. Soc., 140: 1917–1923, 2013. doi: 10.1002/qj.2270. * https://www-miklip.dkrz.de/about/problems/ + """ bin_names = ["category"] - M = forecasts[member_dim].size bin_dim = f"{bin_names[0]}_bin" - # histogram(dim=[]) not allowed therefore add fake member dim - # to apply over when multi-dim observations - if len(observations.dims) == 1: - observations = histogram( - observations, bins=[category_edges], bin_names=bin_names, dim=None - ) - else: - observations = histogram( - observations.expand_dims(member_dim), + M = forecasts[member_dim].size + + def _check_identical_xr_types(a, b): + if type(a) != type(b): + raise ValueError( + f"a and b must be same type, found {type(a)} and {type(b)}" + ) + for d in [a, b]: + if not isinstance(d, (xr.Dataset, xr.DataArray)): + raise ValueError("inputs must be xr.DataArray or xr.Dataset") + + def _check_bin_dim(ds, bin_dim): + """Assert that bin_dim is in ds. Try to guess and rename edges dimension.""" + for d in ["quantile", "threshold", "edge"]: + if d in ds.dims and bin_dim not in ds.dims: + ds = ds.rename({d: bin_dim}) + if bin_dim not in ds.dims: + raise ValueError(f"require {bin_dim} dimension, found {ds.dims}") + return ds + + _check_identical_xr_types(observations, forecasts) + + # different ways of calculating RPS based on category_edges + if isinstance(category_edges, (xr.Dataset, xr.DataArray)) or isinstance( + category_edges, tuple + ): + if isinstance( + category_edges, tuple + ): # edges tuple of two: use for obs and forecast edges separately + observations_edges, forecast_edges = category_edges + _check_identical_xr_types(forecast_edges, forecasts) + _check_identical_xr_types(observations_edges, forecasts) + else: # edges only given once, so use for both obs and forecasts + _check_identical_xr_types(category_edges, forecasts) + observations_edges, forecast_edges = category_edges, category_edges + + # cumulative probs + Fc = (forecasts < category_edges).mean(member_dim) + Oc = observations < category_edges + # todo: mask land + + elif isinstance(category_edges, (np.ndarray, np.array)): + # histogram(dim=[]) not allowed therefore add fake member dim + # to apply over when multi-dim observations + if len(observations.dims) == 1: + observations = histogram( + observations, bins=[category_edges], bin_names=bin_names, dim=None + ) + else: + observations = histogram( + observations.expand_dims(member_dim), + bins=[category_edges], + bin_names=bin_names, + dim=[member_dim], + ) + + forecasts = histogram( + forecasts, bins=[category_edges], bin_names=bin_names, dim=[member_dim], ) + # if fair: + # e = forecasts - forecasts = histogram( - forecasts, - bins=[category_edges], - bin_names=bin_names, - dim=[member_dim], - ) - if fair: - e = forecasts + # normalize f.sum()=1 to make cdf + forecasts = forecasts / forecasts.sum(bin_dim) + observations = observations / observations.sum(bin_dim) - # normalize f.sum()=1 - forecasts = forecasts / forecasts.sum(bin_dim) - observations = observations / observations.sum(bin_dim) + Fc = forecasts.cumsum(bin_dim) + Oc = observations.cumsum(bin_dim) - Fc = forecasts.cumsum(bin_dim) - Oc = observations.cumsum(bin_dim) + elif category_edges is None: # expect cdfs already as inputs + Fc = forecasts + Oc = observations + else: + raise ValueError( + f"category_edges must be xr.DataArray, xr.Dataset, tuple of xr.objects, None or array-like, found {type(category_edges)}" + ) + # check and maybe rename edges dim + Fc = _check_bin_dim(Fc, bin_dim) + Oc = _check_bin_dim(Oc, bin_dim) + + # RPS formulas if fair: - Ec = e.cumsum(bin_dim) + Ec = forecasts.cumsum(bin_dim) res = (((Ec / M) - Oc) ** 2 - Ec * (M - Ec) / (M ** 2 * (M - 1))).sum(bin_dim) else: res = ((Fc - Oc) ** 2).sum(bin_dim) + if weights is not None: res = res.weighted(weights) - res = xr.apply_ufunc(np.clip, res, 0, 1, dask="allowed") # dirty fix return res.mean(dim, keep_attrs=keep_attrs) diff --git a/xskillscore/tests/test_probabilistic.py b/xskillscore/tests/test_probabilistic.py index a7d1f1d7..d668fdb3 100644 --- a/xskillscore/tests/test_probabilistic.py +++ b/xskillscore/tests/test_probabilistic.py @@ -519,15 +519,6 @@ def test_rps_vs_fair_rps(o, f_prob, category_edges, dim): assert (frps <= ufrps).all(), print("fairrps", frps, "\nufrps", ufrps) -@pytest.mark.parametrize("dim", DIMS) -@pytest.mark.parametrize("fair_bool", [True, False]) -def test_rps_limits(o, f_prob, category_edges, fair_bool, dim): - """Test rps between 0 and 1. Note: this only works because np.clip(rps,0,1)""" - res = rps(o, f_prob, dim=dim, fair=fair_bool, category_edges=category_edges) - assert (res <= 1.0).all(), print(res.max()) - assert (res >= 0).all(), print(res.min()) - - @pytest.mark.parametrize( "observation,forecast", [ From 70e1e9be1a7114b8780fb2c747e6c4da89191ae8 Mon Sep 17 00:00:00 2001 From: AS Date: Fri, 26 Feb 2021 15:35:35 +0100 Subject: [PATCH 02/39] add tests --- xskillscore/core/probabilistic.py | 31 ++++++++-- xskillscore/tests/test_probabilistic.py | 82 +++++++++++++++++++++++-- 2 files changed, 102 insertions(+), 11 deletions(-) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index 4d0e6f61..ab479b6d 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -491,6 +491,8 @@ def rps( RPS(p, k) = \\sum_{m=1}^{M} [(\\sum_{k=1}^{m} p_k) - (\\sum_{k=1}^{m} o_k)]^{2} + where ``p`` and ``o`` are forecast and observation probabilities in ``M`` categories. + Parameters ---------- observations : xarray.Dataset or xarray.DataArray @@ -564,6 +566,23 @@ def rps( bin_dim = f"{bin_names[0]}_bin" M = forecasts[member_dim].size + def _bool_to_int(ds): + """convert xr.object of dtype bool to int to evade: + TypeError: numpy boolean subtract, the `-` operator, is not supported""" + + def _helper_bool_to_int(da): + if da.dtype == "bool": + da = da.astype("int") + return da + + if isinstance(ds, xr.Dataset): + ds = ds.map(_helper_bool_to_int) + else: + ds = _helper_bool_to_int(ds) + return ds + + forecasts = _bool_to_int(forecasts) + def _check_identical_xr_types(a, b): if type(a) != type(b): raise ValueError( @@ -599,11 +618,11 @@ def _check_bin_dim(ds, bin_dim): observations_edges, forecast_edges = category_edges, category_edges # cumulative probs - Fc = (forecasts < category_edges).mean(member_dim) - Oc = observations < category_edges + Fc = (forecasts < forecast_edges).mean(member_dim) + Oc = observations < observations_edges # todo: mask land - elif isinstance(category_edges, (np.ndarray, np.array)): + elif isinstance(category_edges, np.ndarray): # histogram(dim=[]) not allowed therefore add fake member dim # to apply over when multi-dim observations if len(observations.dims) == 1: @@ -635,6 +654,8 @@ def _check_bin_dim(ds, bin_dim): Oc = observations.cumsum(bin_dim) elif category_edges is None: # expect cdfs already as inputs + if member_dim in forecasts.dims: + forecasts = forecasts.mean(member_dim) Fc = forecasts Oc = observations else: @@ -648,8 +669,8 @@ def _check_bin_dim(ds, bin_dim): # RPS formulas if fair: - Ec = forecasts.cumsum(bin_dim) - res = (((Ec / M) - Oc) ** 2 - Ec * (M - Ec) / (M ** 2 * (M - 1))).sum(bin_dim) + Ec = Fc * M + res = ((Ec / M - Oc) ** 2 - Ec * (M - Ec) / (M ** 2 * (M - 1))).sum(bin_dim) else: res = ((Fc - Oc) ** 2).sum(bin_dim) diff --git a/xskillscore/tests/test_probabilistic.py b/xskillscore/tests/test_probabilistic.py index d668fdb3..93e22e6b 100644 --- a/xskillscore/tests/test_probabilistic.py +++ b/xskillscore/tests/test_probabilistic.py @@ -484,12 +484,13 @@ def test_rps_wilks_example(): np.testing.assert_allclose(rps(Obs, F2, category_edges), 0.29) -def test_2_category_rps_equals_brier_score(o, f_prob): +@pytest.mark.parametrize("fair_bool", [True, False]) +def test_2_category_rps_equals_brier_score(o, f_prob, fair_bool): """Test that RPS for two categories equals the Brier Score.""" category_edges = np.array([0.0, 0.5, 1.0]) assert_allclose( - rps(o, f_prob, category_edges=category_edges, dim=None), - brier_score(o > 0.5, (f_prob > 0.5).mean("member"), dim=None), + rps(o, f_prob, category_edges=category_edges, dim=None, fair=fair_bool), + brier_score(o > 0.5, (f_prob > 0.5), dim=None, fair=fair_bool), ) @@ -498,6 +499,7 @@ def test_rps_perfect_values(o, category_edges, fair_bool): """Test values for perfect forecast""" f = xr.concat(10 * [o], dim="member") res = rps(o, f, category_edges=category_edges, fair=fair_bool) + print(res) assert (res == 0).all() @@ -512,11 +514,79 @@ def test_rps_dask(o_dask, f_prob_dask, category_edges, fair_bool): @pytest.mark.parametrize("dim", DIMS) def test_rps_vs_fair_rps(o, f_prob, category_edges, dim): - """Test that fair rps is smaller or equal than rps due to ensemble-size - adjustment.""" + """Test that fair rps is smaller (e.g. better) or equal than rps due to ensemble- + size adjustment.""" frps = rps(o, f_prob, dim=dim, fair=True, category_edges=category_edges) ufrps = rps(o, f_prob, dim=dim, fair=False, category_edges=category_edges) - assert (frps <= ufrps).all(), print("fairrps", frps, "\nufrps", ufrps) + # assert (frps <= ufrps).mean() >.9 + assert (frps <= ufrps).all(), print( + "fairrps", + frps, + "\nufrps", + ufrps, + "\n diff: ufrps - frps, should be positive:\n", + ufrps - frps, + ) + + +@pytest.mark.parametrize("fair_bool", [True, False]) +def test_rps_category_edges_xrDataArray(o, f_prob, fair_bool): + """Test rps with category_edges as xrDataArray for forecast and observations edges.""" + actual = rps( + o, + f_prob, + dim="time", + fair=fair_bool, + category_edges=f_prob.quantile(q=[0.3, 0.5, 0.7], dim=["time", "member"]), + ) + assert set(["lon", "lat"]) == set(actual.dims) + assert "quantile" not in actual.dims + + +@pytest.mark.parametrize("fair_bool", [True, False]) +def test_rps_category_edges_xrDataset(o, f_prob, fair_bool): + """Test rps with category_edges as xrDataArray for forecast and observations edges.""" + o = o.to_dataset(name="var") + o["var2"] = o["var"] ** 2 + f_prob = f_prob.to_dataset(name="var") + f_prob["var2"] = f_prob["var"] ** 2 + actual = rps( + o, + f_prob, + dim="time", + fair=fair_bool, + category_edges=f_prob.quantile(q=[0.3, 0.5, 0.7], dim=["time", "member"]), + ) + assert set(["lon", "lat"]) == set(actual.dims) + assert "quantile" not in actual.dims + + +@pytest.mark.parametrize("fair_bool", [True, False]) +def test_rps_category_edges_tuple(o, f_prob, fair_bool): + """Test rps with category_edges as tuple of xrDataArray for forecast and observations edges separately.""" + actual = rps( + o, + f_prob, + dim="time", + fair=fair_bool, + category_edges=( + f_prob.quantile(q=[0.3, 0.5, 0.7], dim=["time", "member"]), + o.quantile(q=[0.3, 0.5, 0.7], dim="time"), + ), + ) + assert set(["lon", "lat"]) == set(actual.dims) + assert "quantile" not in actual.dims + + +@pytest.mark.parametrize("fair_bool", [True, False]) +def test_rps_category_edges_None(o, f_prob, fair_bool): + """Test rps with category_edges as None expecting o and f_prob are already CDFs.""" + edges = xr.DataArray([0.2, 0.4, 0.6, 0.8], dims="quantile") + o_c = o > edges # CDF + f_prob_c = f_prob > edges + actual = rps(o_c, f_prob_c, dim="time", fair=fair_bool, category_edges=None) + assert set(["lon", "lat"]) == set(actual.dims) + assert "quantile" not in actual.dims @pytest.mark.parametrize( From 072f50fb72d86f91f0c2b4b99f244f347ec0afab Mon Sep 17 00:00:00 2001 From: AS Date: Fri, 26 Feb 2021 15:39:28 +0100 Subject: [PATCH 03/39] add tests --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bc19a649..1912e792 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -22,6 +22,8 @@ Features without replacement. (:issue:`215`, :pr:`225`) `Aaron Spring`_ - Added receiver operating characteristic (ROC) :py:func:`~xskillscore.roc`. (:issue:`114`, :issue:`256`, :pr:`236`, :pr:`259`) `Aaron Spring`_ +- Added many options for ``category_edges`` in :py:func:`~xskillscore.rps`, which + allows multi-dimensional edges. (:issue:`275`, :pr:`277`) `Aaron Spring`_ Breaking changes ~~~~~~~~~~~~~~~~ @@ -40,6 +42,8 @@ Bug Fixes (:issue:`255`, :pr:`211`) `Aaron Spring`_ - Passing weights no longer triggers eager computation. (:issue:`218`, :pr:`224`). `Andrew Huang`_ +- :py:func:`~xskillscore.rps` not restricted to ``[0, 1]``. + (:issue:`266`, :pr:`277`) `Aaron Spring`_ Internal Changes ~~~~~~~~~~~~~~~~ From 00d24b20b7dd14ca8b6a216ecd9f139c8c4dd0f5 Mon Sep 17 00:00:00 2001 From: AS Date: Fri, 26 Feb 2021 16:19:40 +0100 Subject: [PATCH 04/39] rtd --- xskillscore/core/probabilistic.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index ab479b6d..8943b20c 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -504,6 +504,7 @@ def rps( The forecast likelihoods of the event. Further requirements are specified based on ``category_edges``. category_edges : array_like, xr.Dataset, xr.DataArray, None + - array_like: Category bin edges used to compute the CDFs based on boolean or logical (True or 1 for event occurance, False or 0 for non-occurance) observations. If ``fair==False``, forecasts should be between 0 and 1 without a dimension @@ -513,10 +514,13 @@ def rps( (True,False) or binary (0, 1) containing dimension ``member_dim``. Similar to np.histogram, all but the last (righthand-most) bin include the left edge and exclude the right edge. The last bin includes both edges. + - xr.Dataset/xr.DataArray: edges of the categories in absolute units provided as dimension ``category_dim``, ``threshold`` or ``quantile``. Forecasts and Observations are also in absolute units. + - tuple of xr.Dataset/xr.DataArray: same as xr.Dataset/xr.DataArray where the first item is taken as category_edges for observations and the second item for category_edges for forecasts + - None: expect than observations and forecasts are already CDFs containing ``category_dim`` dimension dim : str or list of str, optional Dimension over which to compute mean after computing ``rps``. From 9eea7c3a2061f2373398da98d2d2ade4d26ddb5f Mon Sep 17 00:00:00 2001 From: AS Date: Fri, 26 Feb 2021 16:29:59 +0100 Subject: [PATCH 05/39] rtd --- xskillscore/core/probabilistic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index 8943b20c..bf84beb7 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -522,6 +522,7 @@ def rps( is taken as category_edges for observations and the second item for category_edges for forecasts - None: expect than observations and forecasts are already CDFs containing ``category_dim`` dimension + dim : str or list of str, optional Dimension over which to compute mean after computing ``rps``. Defaults to None implying averaging over all dimensions. From ddd51ca46f2f0ae28c4461d306de4060331992ee Mon Sep 17 00:00:00 2001 From: AS Date: Fri, 26 Feb 2021 16:48:16 +0100 Subject: [PATCH 06/39] rtd --- xskillscore/core/probabilistic.py | 37 +++++++++++++++++-------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index bf84beb7..b24f14c0 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -505,23 +505,26 @@ def rps( Further requirements are specified based on ``category_edges``. category_edges : array_like, xr.Dataset, xr.DataArray, None - - array_like: Category bin edges used to compute the CDFs based on boolean or logical - (True or 1 for event occurance, False or 0 for non-occurance) observations. - If ``fair==False``, forecasts should be between 0 and 1 without a dimension - ``member_dim`` or should be boolean (True,False) or binary (0, 1) containing a - member dimension (probabilities will be internally calculated by - ``.mean(member_dim))``. If ``fair==True``, forecasts must be boolean - (True,False) or binary (0, 1) containing dimension ``member_dim``. - Similar to np.histogram, all but the last (righthand-most) bin include the left edge - and exclude the right edge. The last bin includes both edges. - - - xr.Dataset/xr.DataArray: edges of the categories in absolute units provided as dimension - ``category_dim``, ``threshold`` or ``quantile``. Forecasts and Observations are also in absolute units. - - - tuple of xr.Dataset/xr.DataArray: same as xr.Dataset/xr.DataArray where the first item - is taken as category_edges for observations and the second item for category_edges for forecasts - - - None: expect than observations and forecasts are already CDFs containing ``category_dim`` dimension + - array_like: Category bin edges used to compute the CDFs based on boolean or + logical (True or 1 for event occurance, False or 0 for non-occurance) + observations. If ``fair==False``, forecasts should be between 0 and 1 without + a dimension ``member_dim`` or should be boolean (True,False) or binary (0, 1) + containing a member dimension (probabilities will be internally calculated by + ``.mean(member_dim))``. If ``fair==True``, forecasts must be boolean + (True,False) or binary (0, 1) containing dimension ``member_dim``. + Similar to np.histogram, all but the last (righthand-most) bin include the + left edge and exclude the right edge. The last bin includes both edges. + + - xr.Dataset/xr.DataArray: edges of the categories in absolute units provided + as dimension ``category_dim``, ``threshold`` or ``quantile``. Forecasts and + Observations are expected in absolute units. + + - tuple of xr.Dataset/xr.DataArray: same as xr.Dataset/xr.DataArray where the + first item is taken as ``category_edges`` for observations and the second item + for ``category_edges`` for forecasts. + + - None: expect than observations and forecasts are already CDFs containing + ``category_dim`` dimension. dim : str or list of str, optional Dimension over which to compute mean after computing ``rps``. From e0b177af946918b8bdcc3bd3fda60a89a461d920 Mon Sep 17 00:00:00 2001 From: AS Date: Fri, 26 Feb 2021 17:04:59 +0100 Subject: [PATCH 07/39] rtd --- xskillscore/core/probabilistic.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index b24f14c0..60836585 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -628,7 +628,7 @@ def _check_bin_dim(ds, bin_dim): # cumulative probs Fc = (forecasts < forecast_edges).mean(member_dim) Oc = observations < observations_edges - # todo: mask land + # todo: mask Fc and Oc where all nans elif isinstance(category_edges, np.ndarray): # histogram(dim=[]) not allowed therefore add fake member dim @@ -651,8 +651,6 @@ def _check_bin_dim(ds, bin_dim): bin_names=bin_names, dim=[member_dim], ) - # if fair: - # e = forecasts # normalize f.sum()=1 to make cdf forecasts = forecasts / forecasts.sum(bin_dim) From eca1d7ead1694bb9fcb5dcd72803cc7645963ca5 Mon Sep 17 00:00:00 2001 From: AS Date: Fri, 26 Feb 2021 17:10:46 +0100 Subject: [PATCH 08/39] rtd --- xskillscore/core/probabilistic.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index 60836585..05041515 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -491,29 +491,29 @@ def rps( RPS(p, k) = \\sum_{m=1}^{M} [(\\sum_{k=1}^{m} p_k) - (\\sum_{k=1}^{m} o_k)]^{2} - where ``p`` and ``o`` are forecast and observation probabilities in ``M`` categories. + where ``p`` and ``o`` are forecast and observation probabilities in ``M`` + categories. Parameters ---------- observations : xarray.Dataset or xarray.DataArray The observations or set of observations of the event. Further requirements are specified based on ``category_edges``. - Data should be boolean or logical \ - (True or 1 for event occurance, False or 0 for non-occurance). forecasts : xarray.Dataset or xarray.DataArray The forecast likelihoods of the event. Further requirements are specified based on ``category_edges``. category_edges : array_like, xr.Dataset, xr.DataArray, None - - array_like: Category bin edges used to compute the CDFs based on boolean or - logical (True or 1 for event occurance, False or 0 for non-occurance) - observations. If ``fair==False``, forecasts should be between 0 and 1 without - a dimension ``member_dim`` or should be boolean (True,False) or binary (0, 1) - containing a member dimension (probabilities will be internally calculated by - ``.mean(member_dim))``. If ``fair==True``, forecasts must be boolean - (True,False) or binary (0, 1) containing dimension ``member_dim``. - Similar to np.histogram, all but the last (righthand-most) bin include the + - array_like: Category bin edges used to compute the CDFs. Similar to + np.histogram, all but the last (righthand-most) bin include the left edge and exclude the right edge. The last bin includes both edges. + CDFs based on boolean or logical (True or 1 for event occurance, False or 0 + for non-occurance) observations. + If ``fair==False``, forecasts should be between 0 and 1 without a dimension + ``member_dim`` or boolean / binary containing a member dimension + (probabilities will be internally calculated by ``.mean(member_dim))``. + If ``fair==True``, forecasts must be boolean / binary containing dimension + ``member_dim``. - xr.Dataset/xr.DataArray: edges of the categories in absolute units provided as dimension ``category_dim``, ``threshold`` or ``quantile``. Forecasts and @@ -524,7 +524,7 @@ def rps( for ``category_edges`` for forecasts. - None: expect than observations and forecasts are already CDFs containing - ``category_dim`` dimension. + ``category_dim``, ``quantile``, ``edge`` or ``threshold`` dimension. dim : str or list of str, optional Dimension over which to compute mean after computing ``rps``. From 09b1e866dec9c209f7d8a9ffaefd131b6c9c2b75 Mon Sep 17 00:00:00 2001 From: AS Date: Fri, 26 Feb 2021 17:25:14 +0100 Subject: [PATCH 09/39] test category_edges np.array or xr.DataArray same results --- xskillscore/tests/test_probabilistic.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/xskillscore/tests/test_probabilistic.py b/xskillscore/tests/test_probabilistic.py index 93e22e6b..c4701b8f 100644 --- a/xskillscore/tests/test_probabilistic.py +++ b/xskillscore/tests/test_probabilistic.py @@ -473,15 +473,24 @@ def test_rps_wilks_example(): """ category_edges = np.array([-0.01, 0.01, 0.24, 10]) # first example + # xhistogram way with np.array category_edges Obs = xr.DataArray([0.0001]) # no precip F1 = xr.DataArray([0] * 2 + [0.1] * 5 + [0.3] * 3, dims="member") F2 = xr.DataArray([0] * 2 + [0.1] * 3 + [0.3] * 5, dims="member") np.testing.assert_allclose(rps(Obs, F1, category_edges), 0.73) np.testing.assert_allclose(rps(Obs, F2, category_edges), 0.89) + # xr way with xr.DataArray category_edges + xr_category_edges = xr.DataArray(category_edges, dims="quantile") + assert_allclose(rps(Obs, F1, category_edges), rps(Obs, F1, xr_category_edges)) + assert_allclose(rps(Obs, F2, category_edges), rps(Obs, F2, xr_category_edges)) + # second example Obs = xr.DataArray([0.3]) # larger than 0.25 np.testing.assert_allclose(rps(Obs, F1, category_edges), 0.53) np.testing.assert_allclose(rps(Obs, F2, category_edges), 0.29) + # xr way with xr.DataArray category_edges + assert_allclose(rps(Obs, F1, category_edges), rps(Obs, F1, xr_category_edges)) + assert_allclose(rps(Obs, F2, category_edges), rps(Obs, F2, xr_category_edges)) @pytest.mark.parametrize("fair_bool", [True, False]) @@ -499,7 +508,6 @@ def test_rps_perfect_values(o, category_edges, fair_bool): """Test values for perfect forecast""" f = xr.concat(10 * [o], dim="member") res = rps(o, f, category_edges=category_edges, fair=fair_bool) - print(res) assert (res == 0).all() @@ -518,15 +526,7 @@ def test_rps_vs_fair_rps(o, f_prob, category_edges, dim): size adjustment.""" frps = rps(o, f_prob, dim=dim, fair=True, category_edges=category_edges) ufrps = rps(o, f_prob, dim=dim, fair=False, category_edges=category_edges) - # assert (frps <= ufrps).mean() >.9 - assert (frps <= ufrps).all(), print( - "fairrps", - frps, - "\nufrps", - ufrps, - "\n diff: ufrps - frps, should be positive:\n", - ufrps - frps, - ) + assert (frps <= ufrps).all() @pytest.mark.parametrize("fair_bool", [True, False]) From 51cb1b0e95f4e5f07fbdf9990cefabd97a6e9629 Mon Sep 17 00:00:00 2001 From: AS Date: Sun, 28 Feb 2021 14:32:06 +0100 Subject: [PATCH 10/39] mask all nans --- xskillscore/core/probabilistic.py | 19 +++++++++++++++++-- xskillscore/tests/test_probabilistic.py | 18 ++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index 05041515..1088bf40 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -609,6 +609,15 @@ def _check_bin_dim(ds, bin_dim): raise ValueError(f"require {bin_dim} dimension, found {ds.dims}") return ds + def _keep_nans_masked(ds_before, ds_after, dim=None, ignore=None): + """Preserve NaNs where all over dim were NaNs. Don't consider dimensions ignore when finding all NaNs.""" + mask = ds_before.isnull().all(dim) + overlap_dims = set(mask.dims) & set(ignore) + if len(overlap_dims) > 0: + mask = mask.mean(overlap_dims) + ds_after = ds_after.where(~mask.astype("bool"), other=np.nan) + return ds_after + _check_identical_xr_types(observations, forecasts) # different ways of calculating RPS based on category_edges @@ -628,7 +637,6 @@ def _check_bin_dim(ds, bin_dim): # cumulative probs Fc = (forecasts < forecast_edges).mean(member_dim) Oc = observations < observations_edges - # todo: mask Fc and Oc where all nans elif isinstance(category_edges, np.ndarray): # histogram(dim=[]) not allowed therefore add fake member dim @@ -682,7 +690,14 @@ def _check_bin_dim(ds, bin_dim): if weights is not None: res = res.weighted(weights) - return res.mean(dim, keep_attrs=keep_attrs) + res = res.mean(dim, keep_attrs=keep_attrs) + + # keep nans and prevent 0 for all nan grids + print(observations.dims, res.dims, dim) + res = _keep_nans_masked( + observations, res, dim, ignore=["quantile", "threshold", "edge", "category_bin"] + ) + return res def rank_histogram(observations, forecasts, dim=None, member_dim="member"): diff --git a/xskillscore/tests/test_probabilistic.py b/xskillscore/tests/test_probabilistic.py index c4701b8f..399d24f0 100644 --- a/xskillscore/tests/test_probabilistic.py +++ b/xskillscore/tests/test_probabilistic.py @@ -589,6 +589,24 @@ def test_rps_category_edges_None(o, f_prob, fair_bool): assert "quantile" not in actual.dims +@pytest.mark.parametrize( + "category_edges", + [ + xr.DataArray([0.2, 0.4, 0.6, 0.8], dims="quantile"), + np.array([0.2, 0.4, 0.6, 0.8]), + ], +) +@pytest.mark.parametrize("fair_bool", [True, False]) +def test_rps_keeps_masked(o, f_prob, fair_bool, category_edges): + """Test rps keeps NaNs.""" + o = o.where(o.lat > 1) + f_prob = f_prob.where(f_prob.lat > 1) + actual = rps(o, f_prob, dim="time", category_edges=category_edges) + assert set(["lon", "lat"]) == set(actual.dims) + assert actual.isel(lat=[0, 1]).isnull().all() + assert actual.isel(lat=slice(2, None)).notnull().all() + + @pytest.mark.parametrize( "observation,forecast", [ From 35396edcaf19cc0db9133bcd476f4cdb86fd3b88 Mon Sep 17 00:00:00 2001 From: AS Date: Sun, 28 Feb 2021 15:36:24 +0100 Subject: [PATCH 11/39] add Weigel ref --- xskillscore/core/probabilistic.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index 1088bf40..b2658ee6 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -488,12 +488,16 @@ def rps( """Calculate Ranked Probability Score. .. math:: - RPS(p, k) = \\sum_{m=1}^{M} - [(\\sum_{k=1}^{m} p_k) - (\\sum_{k=1}^{m} o_k)]^{2} + RPS = \\sum_{m=1}^{M}[(\\sum_{k=1}^{m} y_k) - (\\sum_{k=1}^{m} o_k)]^{2} - where ``p`` and ``o`` are forecast and observation probabilities in ``M`` + where ``y`` and ``o`` are forecast and observation probabilities in ``M`` categories. + .. note:: + Takes the sum over all categories as in Weigel et al. 2007 and not the mean as + in https://www.cawcr.gov.au/projects/verification/verif_web_page.html#RPS. + Therefore RPS has no upper boundary. + Parameters ---------- observations : xarray.Dataset or xarray.DataArray @@ -564,7 +568,9 @@ def rps( References ---------- - * https://www.cawcr.gov.au/projects/verification/verif_web_page.html#RPS + * Weigel, A. P., Liniger, M. A., & Appenzeller, C. (2007). The Discrete Brier and + Ranked Probability Skill Scores. Monthly Weather Review, 135(1), 118–124. + doi: 10/b59qz5 * C. A. T. Ferro. Fair scores for ensemble forecasts. Q.R.J. Meteorol. Soc., 140: 1917–1923, 2013. doi: 10.1002/qj.2270. * https://www-miklip.dkrz.de/about/problems/ From 724853dd165524e37db635c4416634b27ea780ca Mon Sep 17 00:00:00 2001 From: AS Date: Sun, 28 Feb 2021 17:00:07 +0100 Subject: [PATCH 12/39] move helper functions out of rps --- xskillscore/core/probabilistic.py | 55 ++++++++----------------------- xskillscore/core/utils.py | 35 ++++++++++++++++++++ 2 files changed, 48 insertions(+), 42 deletions(-) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index b2658ee6..60460325 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -7,8 +7,11 @@ from .contingency import Contingency from .utils import ( _add_as_coord, + _bool_to_int, + _check_identical_xr_types, _fail_if_dim_empty, _get_bin_centers, + _keep_nans_masked, _preprocess_dims, _stack_input_if_needed, histogram, @@ -475,6 +478,16 @@ def threshold_brier_score( return res.mean(dim, keep_attrs=keep_attrs) +def _check_bin_dim(ds, bin_dim): + """Assert that bin_dim is in ds. Try to guess and rename edges dimension.""" + for d in ["quantile", "threshold", "edge"]: + if d in ds.dims and bin_dim not in ds.dims: + ds = ds.rename({d: bin_dim}) + if bin_dim not in ds.dims: + raise ValueError(f"require {bin_dim} dimension, found {ds.dims}") + return ds + + def rps( observations, forecasts, @@ -580,50 +593,8 @@ def rps( bin_dim = f"{bin_names[0]}_bin" M = forecasts[member_dim].size - def _bool_to_int(ds): - """convert xr.object of dtype bool to int to evade: - TypeError: numpy boolean subtract, the `-` operator, is not supported""" - - def _helper_bool_to_int(da): - if da.dtype == "bool": - da = da.astype("int") - return da - - if isinstance(ds, xr.Dataset): - ds = ds.map(_helper_bool_to_int) - else: - ds = _helper_bool_to_int(ds) - return ds - forecasts = _bool_to_int(forecasts) - def _check_identical_xr_types(a, b): - if type(a) != type(b): - raise ValueError( - f"a and b must be same type, found {type(a)} and {type(b)}" - ) - for d in [a, b]: - if not isinstance(d, (xr.Dataset, xr.DataArray)): - raise ValueError("inputs must be xr.DataArray or xr.Dataset") - - def _check_bin_dim(ds, bin_dim): - """Assert that bin_dim is in ds. Try to guess and rename edges dimension.""" - for d in ["quantile", "threshold", "edge"]: - if d in ds.dims and bin_dim not in ds.dims: - ds = ds.rename({d: bin_dim}) - if bin_dim not in ds.dims: - raise ValueError(f"require {bin_dim} dimension, found {ds.dims}") - return ds - - def _keep_nans_masked(ds_before, ds_after, dim=None, ignore=None): - """Preserve NaNs where all over dim were NaNs. Don't consider dimensions ignore when finding all NaNs.""" - mask = ds_before.isnull().all(dim) - overlap_dims = set(mask.dims) & set(ignore) - if len(overlap_dims) > 0: - mask = mask.mean(overlap_dims) - ds_after = ds_after.where(~mask.astype("bool"), other=np.nan) - return ds_after - _check_identical_xr_types(observations, forecasts) # different ways of calculating RPS based on category_edges diff --git a/xskillscore/core/utils.py b/xskillscore/core/utils.py index 80e26c06..dce22978 100644 --- a/xskillscore/core/utils.py +++ b/xskillscore/core/utils.py @@ -164,3 +164,38 @@ def histogram(*args, bins=None, bin_names=None, **kwargs): if bin_names: args = (arg.rename(bin_names[i]) for i, arg in enumerate(args)) return xhist(*args, bins=bins, **kwargs) + + +def _bool_to_int(ds): + """convert xr.object of dtype bool to int to evade: + TypeError: numpy boolean subtract, the `-` operator, is not supported""" + + def _helper_bool_to_int(da): + if da.dtype == "bool": + da = da.astype("int") + return da + + if isinstance(ds, xr.Dataset): + ds = ds.map(_helper_bool_to_int) + else: + ds = _helper_bool_to_int(ds) + return ds + + +def _check_identical_xr_types(a, b): + """Check that a and b are both xr.Dataset or both xr.DataArray.""" + if type(a) != type(b): + raise ValueError(f"a and b must be same type, found {type(a)} and {type(b)}") + for d in [a, b]: + if not isinstance(d, (xr.Dataset, xr.DataArray)): + raise ValueError("inputs must be xr.DataArray or xr.Dataset") + + +def _keep_nans_masked(ds_before, ds_after, dim=None, ignore=None): + """Preserve all NaNs from ds_before for ds_after over while ignoring some dimensions optionally.""" + mask = ds_before.isnull().all(dim) + overlap_dims = set(mask.dims) & set(ignore) + if len(overlap_dims) > 0: + mask = mask.mean(overlap_dims) + ds_after = ds_after.where(~mask.astype("bool"), other=np.nan) + return ds_after From 0e62ec58a2d9850a7e35c7f6b44d37cded61b9bf Mon Sep 17 00:00:00 2001 From: AS Date: Mon, 1 Mar 2021 15:06:34 +0100 Subject: [PATCH 13/39] working version --- xskillscore/core/contingency.py | 2 + xskillscore/core/probabilistic.py | 137 +++++++++++++++++++----- xskillscore/core/utils.py | 66 +++++++++++- xskillscore/tests/test_probabilistic.py | 93 ++++++++++++---- 4 files changed, 242 insertions(+), 56 deletions(-) diff --git a/xskillscore/core/contingency.py b/xskillscore/core/contingency.py index ae00f09c..60750dd3 100644 --- a/xskillscore/core/contingency.py +++ b/xskillscore/core/contingency.py @@ -13,6 +13,8 @@ def _get_category_bounds(category_edges): """Return formatted string of category bounds given list of category edges""" + if isinstance(category_edges, (xr.DataArray, xr.Dataset)): + category_edges = category_edges.category_edge.values bounds = [ f"[{str(category_edges[i])}, {str(category_edges[i + 1])})" for i in range(len(category_edges) - 2) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index 60460325..1dff4bbd 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -478,16 +478,43 @@ def threshold_brier_score( return res.mean(dim, keep_attrs=keep_attrs) -def _check_bin_dim(ds, bin_dim): +def _check_bin_dim(ds): """Assert that bin_dim is in ds. Try to guess and rename edges dimension.""" - for d in ["quantile", "threshold", "edge"]: - if d in ds.dims and bin_dim not in ds.dims: - ds = ds.rename({d: bin_dim}) + bin_dim = "category_edge" if bin_dim not in ds.dims: raise ValueError(f"require {bin_dim} dimension, found {ds.dims}") return ds +def _check_data_within_edges(forecasts, forecasts_edges): + """Check that forecasts_edges has category_edge dimension and forecasts_edges span range of forecasts (per variable if xr.Dataset).""" + if "category_edge" not in forecasts_edges.dims: + raise ValueError( + f"Expect to find category_edge in forecast_edges, found {forecasts_edges.dims}" + ) + + def _check(forecasts, forecasts_edges): + if forecasts.min() < forecasts_edges.min(): + raise ValueError( + f"found forecasts outside forecast_edges, found forecasts_edges.min() = \n{forecasts_edges.min()}\n and forecasts.min() = \n {forecasts.min()}" + ) + if forecasts.max() > forecasts_edges.max(): + raise ValueError( + f"found forecasts outside forecast_edges, found forecasts_edges.max() = \n{forecasts_edges.max()}\n and forecasts.max() = \n {forecasts.max()}" + ) + + if isinstance(forecasts, xr.Dataset): + for v in forecasts.data_vars: + _check(forecasts[v], forecasts_edges[v]) + elif isinstance(forecasts, xr.DataArray): + _check(forecasts, forecasts_edges) + else: + raise ValueError("only defined for xr.DataArrays and xr.Datasets") + + +from .contingency import _get_category_bounds + + def rps( observations, forecasts, @@ -497,6 +524,7 @@ def rps( weights=None, keep_attrs=False, member_dim="member", + # skipna=False, ): """Calculate Ranked Probability Score. @@ -533,15 +561,14 @@ def rps( ``member_dim``. - xr.Dataset/xr.DataArray: edges of the categories in absolute units provided - as dimension ``category_dim``, ``threshold`` or ``quantile``. Forecasts and - Observations are expected in absolute units. + as dimension ``category_edges``. These edges must span the full range of the observations and forecasts distribution. Forecasts and observations are expected in absolute units. - - tuple of xr.Dataset/xr.DataArray: same as xr.Dataset/xr.DataArray where the + - tuple of xr.Dataset/xr.DataArray: same as xr.Dataset/xr.DataArray, where the first item is taken as ``category_edges`` for observations and the second item for ``category_edges`` for forecasts. - None: expect than observations and forecasts are already CDFs containing - ``category_dim``, ``quantile``, ``edge`` or ``threshold`` dimension. + ``category_edges`` dimension. dim : str or list of str, optional Dimension over which to compute mean after computing ``rps``. @@ -590,7 +617,7 @@ def rps( """ bin_names = ["category"] - bin_dim = f"{bin_names[0]}_bin" + bin_dim = f"{bin_names[0]}_edge" M = forecasts[member_dim].size forecasts = _bool_to_int(forecasts) @@ -604,47 +631,86 @@ def rps( if isinstance( category_edges, tuple ): # edges tuple of two: use for obs and forecast edges separately - observations_edges, forecast_edges = category_edges - _check_identical_xr_types(forecast_edges, forecasts) + observations_edges, forecasts_edges = category_edges + _check_identical_xr_types(forecasts_edges, forecasts) _check_identical_xr_types(observations_edges, forecasts) else: # edges only given once, so use for both obs and forecasts _check_identical_xr_types(category_edges, forecasts) - observations_edges, forecast_edges = category_edges, category_edges + observations_edges, forecasts_edges = category_edges, category_edges + + _check_data_within_edges(forecasts, forecasts_edges) + _check_data_within_edges(observations, observations_edges) - # cumulative probs - Fc = (forecasts < forecast_edges).mean(member_dim) - Oc = observations < observations_edges + # cumulative probs, ignore lowest threshold as below category_edges + Fc = (forecasts < forecasts_edges).mean("member").diff(bin_dim).cumsum(bin_dim) + Oc = ( + (observations < observations_edges) + .astype("int") + .isel({bin_dim: slice(1, None)}) + ) + + def _check_CDF(cdf): + # CDF <=1 + assert (Fc <= 1.0).all(), print(Fc) + # CDF >=0 + assert (Fc >= 0.0).all(), print(Fc) + # CDF monotonic increasing + assert (Fc.diff("category_edge") >= 0).all() + + _check_CDF(Fc) + _check_CDF(Oc) + + Fc[bin_dim] = _get_category_bounds(forecasts_edges) + Oc[bin_dim] = _get_category_bounds(observations_edges) elif isinstance(category_edges, np.ndarray): + # category_edges are 1d array and probability edges + # check monotonic increasing + if np.all(np.diff(category_edges) < 0): + raise ValueError(f"must be monotonic increasing, found {category_edges}") + if category_edges.min() < 0 or category_edges.max() > 1: + raise ValueError(f"must be in [0, 1], found {category_edges}") # histogram(dim=[]) not allowed therefore add fake member dim # to apply over when multi-dim observations if len(observations.dims) == 1: - observations = histogram( - observations, bins=[category_edges], bin_names=bin_names, dim=None + observations_bins = histogram( + observations, + bins=[category_edges], + bin_names=["category_edge"], + dim=None, ) else: - observations = histogram( + observations_bins = histogram( observations.expand_dims(member_dim), bins=[category_edges], - bin_names=bin_names, + bin_names=["category_edge"], dim=[member_dim], + ) # .squeeze() + if "category_edge_bin" in observations_bins.dims: + observations_bins = observations_bins.rename( + {"category_edge_bin": "category_edge"} ) forecasts = histogram( forecasts, bins=[category_edges], - bin_names=bin_names, + # bin_names=bin_names, + bin_names=["category_edge"], dim=[member_dim], ) + if "category_edge_bin" in forecasts.dims: + forecasts = forecasts.rename({"category_edge_bin": "category_edge"}) # normalize f.sum()=1 to make cdf forecasts = forecasts / forecasts.sum(bin_dim) - observations = observations / observations.sum(bin_dim) Fc = forecasts.cumsum(bin_dim) - Oc = observations.cumsum(bin_dim) + Oc = observations_bins.cumsum(bin_dim) + + Fc[bin_dim] = _get_category_bounds(category_edges) + Oc[bin_dim] = _get_category_bounds(category_edges) - elif category_edges is None: # expect cdfs already as inputs + elif category_edges is None: # expect CDFs already as inputs if member_dim in forecasts.dims: forecasts = forecasts.mean(member_dim) Fc = forecasts @@ -655,8 +721,11 @@ def rps( ) # check and maybe rename edges dim - Fc = _check_bin_dim(Fc, bin_dim) - Oc = _check_bin_dim(Oc, bin_dim) + Fc = _check_bin_dim(Fc) + Oc = _check_bin_dim(Oc) + + assert (Fc <= 1.0).all(), print("Fc > 1", Fc) + assert (Oc <= 1.0).all(), print("Oc > 1", Oc) # RPS formulas if fair: @@ -665,15 +734,25 @@ def rps( else: res = ((Fc - Oc) ** 2).sum(bin_dim) + # add bin edges as coords + res = res.assign_coords( + {"forecasts_category_edge": ", ".join(_get_category_bounds(Fc[bin_dim].values))} + ) + res = res.assign_coords( + { + "observations_category_edge": ", ".join( + _get_category_bounds(Oc[bin_dim].values) + ) + } + ) + if weights is not None: res = res.weighted(weights) + res = res.mean(dim, keep_attrs=keep_attrs) # keep nans and prevent 0 for all nan grids - print(observations.dims, res.dims, dim) - res = _keep_nans_masked( - observations, res, dim, ignore=["quantile", "threshold", "edge", "category_bin"] - ) + res = _keep_nans_masked(observations, res, dim, ignore=["category_edge"]) return res diff --git a/xskillscore/core/utils.py b/xskillscore/core/utils.py index dce22978..01b95cf6 100644 --- a/xskillscore/core/utils.py +++ b/xskillscore/core/utils.py @@ -193,9 +193,67 @@ def _check_identical_xr_types(a, b): def _keep_nans_masked(ds_before, ds_after, dim=None, ignore=None): """Preserve all NaNs from ds_before for ds_after over while ignoring some dimensions optionally.""" - mask = ds_before.isnull().all(dim) - overlap_dims = set(mask.dims) & set(ignore) - if len(overlap_dims) > 0: - mask = mask.mean(overlap_dims) + print( + "ds_before.dims =", + ds_before.dims, + "ds_after.dims", + ds_after.dims, + " dim =", + dim, + ) + if dim is None: + dim = list(ds_before.dims) + elif isinstance(dim, str): + dim = [dim] + print("dim =", dim) + + if ignore is None: + ignore = [] + elif isinstance(ignore, str): + ignore = list(ignore) + + # dim = set(dim) - set(ignore) + + # all_dim = ds_before.dims #[d for d in dim if d in ds_before.dims] + all_dim = set(dim) ^ set(ignore) + print("all_dim", all_dim) + all_dim = [d for d in all_dim if d in ds_before.dims] + print("all_dim", all_dim) + + mask = ds_before.isnull().all(all_dim) + print("mask", mask) + if False: + for d in dim: + assert d not in mask + if ignore is not None: + overlap_dims = set(mask.dims) & set(ignore) + print("overlap_dims", overlap_dims) + if len(overlap_dims) > 0: + mask = mask.mean(overlap_dims) + print("mask.dims", mask.dims) ds_after = ds_after.where(~mask.astype("bool"), other=np.nan) + for d in dim: + assert d not in ds_after.dims + if ignore is not None: + for d in ignore: + assert d not in ds_after.dims return ds_after + + +def quantile_edges_pad_lower_upper( + ds, edges, dim, lower=0, upper=1, category_edge_dim="category_edge" +): + assert isinstance(dim, list) + if isinstance(edges, list): + edges = np.array(edges) + assert isinstance(edges, np.ndarray) + q_edges = ds.quantile(q=edges, dim=dim).rename({"quantile": category_edge_dim}) + ds_dim_reduced = ds.isel({d: 0 for d in dim}, drop=True) + lower = lower * xr.ones_like(ds_dim_reduced).expand_dims( + category_edge_dim + ).assign_coords({category_edge_dim: [lower]}) + upper = upper * xr.ones_like(ds_dim_reduced).expand_dims( + category_edge_dim + ).assign_coords({category_edge_dim: [upper]}) + ds_pad = xr.concat([lower, q_edges, upper], dim=category_edge_dim) + return ds_pad diff --git a/xskillscore/tests/test_probabilistic.py b/xskillscore/tests/test_probabilistic.py index 399d24f0..e2b234da 100644 --- a/xskillscore/tests/test_probabilistic.py +++ b/xskillscore/tests/test_probabilistic.py @@ -21,7 +21,7 @@ rps, threshold_brier_score, ) -from xskillscore.core.utils import suppress_warnings +from xskillscore.core.utils import quantile_edges_pad_lower_upper, suppress_warnings DIMS = ["lon", "lat", ["lon", "lat"], None, []] @@ -471,16 +471,22 @@ def test_rps_wilks_example(): """Test with values from Wilks, D. S. (2006). Statistical methods in the atmospheric sciences (2nd ed, Vol. 91). Amsterdam ; Boston: Academic Press. p.301. """ - category_edges = np.array([-0.01, 0.01, 0.24, 10]) + category_edges = np.array([0.0, 0.01, 0.24, 1.0]) # first example # xhistogram way with np.array category_edges - Obs = xr.DataArray([0.0001]) # no precip - F1 = xr.DataArray([0] * 2 + [0.1] * 5 + [0.3] * 3, dims="member") - F2 = xr.DataArray([0] * 2 + [0.1] * 3 + [0.3] * 5, dims="member") + Obs = xr.DataArray([0.0001]) # .expand_dims('time') # no precip + F1 = xr.DataArray( + [0] * 2 + [0.1] * 5 + [0.3] * 3, dims="member" + ) # .expand_dims('time') + F2 = xr.DataArray( + [0] * 2 + [0.1] * 3 + [0.3] * 5, dims="member" + ) # .expand_dims('time') np.testing.assert_allclose(rps(Obs, F1, category_edges), 0.73) np.testing.assert_allclose(rps(Obs, F2, category_edges), 0.89) # xr way with xr.DataArray category_edges - xr_category_edges = xr.DataArray(category_edges, dims="quantile") + xr_category_edges = xr.DataArray( + category_edges, dims="category_edge", coords={"category_edge": category_edges} + ) assert_allclose(rps(Obs, F1, category_edges), rps(Obs, F1, xr_category_edges)) assert_allclose(rps(Obs, F2, category_edges), rps(Obs, F2, xr_category_edges)) @@ -498,7 +504,9 @@ def test_2_category_rps_equals_brier_score(o, f_prob, fair_bool): """Test that RPS for two categories equals the Brier Score.""" category_edges = np.array([0.0, 0.5, 1.0]) assert_allclose( - rps(o, f_prob, category_edges=category_edges, dim=None, fair=fair_bool), + rps(o, f_prob, category_edges=category_edges, dim=None, fair=fair_bool).drop( + ["forecasts_category_edge", "observations_category_edge"] + ), brier_score(o > 0.5, (f_prob > 0.5), dim=None, fair=fair_bool), ) @@ -532,15 +540,18 @@ def test_rps_vs_fair_rps(o, f_prob, category_edges, dim): @pytest.mark.parametrize("fair_bool", [True, False]) def test_rps_category_edges_xrDataArray(o, f_prob, fair_bool): """Test rps with category_edges as xrDataArray for forecast and observations edges.""" + category_edges = quantile_edges_pad_lower_upper( + f_prob, edges=[0.2, 0.4, 0.6, 0.8], dim=["time", "member"] + ) actual = rps( o, f_prob, dim="time", fair=fair_bool, - category_edges=f_prob.quantile(q=[0.3, 0.5, 0.7], dim=["time", "member"]), + category_edges=category_edges, ) assert set(["lon", "lat"]) == set(actual.dims) - assert "quantile" not in actual.dims + assert "category_edge" not in actual.dims @pytest.mark.parametrize("fair_bool", [True, False]) @@ -550,40 +561,45 @@ def test_rps_category_edges_xrDataset(o, f_prob, fair_bool): o["var2"] = o["var"] ** 2 f_prob = f_prob.to_dataset(name="var") f_prob["var2"] = f_prob["var"] ** 2 + category_edges = quantile_edges_pad_lower_upper( + f_prob, edges=[0.2, 0.4, 0.6, 0.8], dim=["time", "member"] + ) actual = rps( o, f_prob, dim="time", fair=fair_bool, - category_edges=f_prob.quantile(q=[0.3, 0.5, 0.7], dim=["time", "member"]), + category_edges=category_edges, ) assert set(["lon", "lat"]) == set(actual.dims) - assert "quantile" not in actual.dims + assert "category_edge" not in actual.dims @pytest.mark.parametrize("fair_bool", [True, False]) def test_rps_category_edges_tuple(o, f_prob, fair_bool): """Test rps with category_edges as tuple of xrDataArray for forecast and observations edges separately.""" + edges = [0.3, 0.5, 0.7] + o_edges = quantile_edges_pad_lower_upper(o, edges=edges, dim=["time"]) + f_edges = quantile_edges_pad_lower_upper( + f_prob, edges=edges, dim=["time", "member"] + ) actual = rps( o, f_prob, dim="time", fair=fair_bool, - category_edges=( - f_prob.quantile(q=[0.3, 0.5, 0.7], dim=["time", "member"]), - o.quantile(q=[0.3, 0.5, 0.7], dim="time"), - ), + category_edges=(o_edges, f_edges), ) assert set(["lon", "lat"]) == set(actual.dims) - assert "quantile" not in actual.dims + assert "category_edge" not in actual.dims @pytest.mark.parametrize("fair_bool", [True, False]) def test_rps_category_edges_None(o, f_prob, fair_bool): """Test rps with category_edges as None expecting o and f_prob are already CDFs.""" - edges = xr.DataArray([0.2, 0.4, 0.6, 0.8], dims="quantile") + edges = xr.DataArray([0, 0.2, 0.4, 0.6, 0.8, 1.0], dims="category_edge") o_c = o > edges # CDF - f_prob_c = f_prob > edges + f_prob_c = f_prob > edges # CDF actual = rps(o_c, f_prob_c, dim="time", fair=fair_bool, category_edges=None) assert set(["lon", "lat"]) == set(actual.dims) assert "quantile" not in actual.dims @@ -592,19 +608,50 @@ def test_rps_category_edges_None(o, f_prob, fair_bool): @pytest.mark.parametrize( "category_edges", [ - xr.DataArray([0.2, 0.4, 0.6, 0.8], dims="quantile"), - np.array([0.2, 0.4, 0.6, 0.8]), + xr.DataArray([0, 0.2, 0.4, 0.6, 0.8, 1.0], dims="category_edge"), + np.array([0, 0.2, 0.4, 0.6, 0.8, 1.0]), ], + ids=["edge xr", "edge np"], ) -@pytest.mark.parametrize("fair_bool", [True, False]) +@pytest.mark.parametrize("fair_bool", [True, False], ids=["fair=True", "fair=False"]) def test_rps_keeps_masked(o, f_prob, fair_bool, category_edges): """Test rps keeps NaNs.""" o = o.where(o.lat > 1) f_prob = f_prob.where(f_prob.lat > 1) + print("o", o) + print("f_prob", f_prob) actual = rps(o, f_prob, dim="time", category_edges=category_edges) assert set(["lon", "lat"]) == set(actual.dims) - assert actual.isel(lat=[0, 1]).isnull().all() - assert actual.isel(lat=slice(2, None)).notnull().all() + assert actual.isel(lat=[0, 1]).isnull().all(), print(actual.isel(lat=[0, 1])) + assert actual.isel(lat=slice(2, None)).notnull().all(), print( + actual.isel(lat=slice(2, None)) + ) + + +@pytest.mark.parametrize("fair_bool", [True, False], ids=["bool=fair", "fair=False"]) +def test_rps_new_identical_old_xhistogram(o, f_prob, fair_bool): + """Test that new rps algorithm is identical to old algorithm with xhistogram. + Makes a difference whether full range of f_prob is covered or not.""" + category_edges_np = np.array([0, 0.2, 0.4, 0.6, 0.8, 1.0]) + category_edges_xr = xr.DataArray( + category_edges_np, + dims="category_edge", + coords={"category_edge": category_edges_np}, + ) + dim = "time" + f_prob = f_prob.isel(lon=2, time=slice(None, 3), drop=True) + o = o.isel(lon=2, time=slice(None, 3), drop=True) + print("member", f_prob.member.size) + print("f_prob", f_prob) + print("o", o) + print("category_edges_xr") + actual = rps(o, f_prob, dim=dim, category_edges=category_edges_xr) + print("\n\n category_edges_np") + expected = rps(o, f_prob, dim=dim, category_edges=category_edges_np) + print("actual", actual) + print("expected", expected) + assert_allclose(actual.rename("histogram_category_edge"), expected) + # assert False @pytest.mark.parametrize( From b2352b22fd539e6e1826797f4d0327f8cb6fab04 Mon Sep 17 00:00:00 2001 From: AS Date: Mon, 1 Mar 2021 16:47:03 +0100 Subject: [PATCH 14/39] refactor rps_xhist to tests --- xskillscore/core/probabilistic.py | 177 +++++++++++------------- xskillscore/tests/test_probabilistic.py | 133 +++++++++++++++--- 2 files changed, 195 insertions(+), 115 deletions(-) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index 1dff4bbd..47970d11 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -542,40 +542,37 @@ def rps( Parameters ---------- observations : xarray.Dataset or xarray.DataArray - The observations or set of observations of the event. + The observations of the event. Further requirements are specified based on ``category_edges``. forecasts : xarray.Dataset or xarray.DataArray - The forecast likelihoods of the event. + The forecast of the event. Further requirements are specified based on ``category_edges``. category_edges : array_like, xr.Dataset, xr.DataArray, None + Category edges used to compute the CDFs. Similar to np.histogram, all but the last (righthand-most) bin include the left edge and exclude the right edge. The last bin includes both edges. + These ``category_edge`` must span the full range of the observations and forecasts distribution. Forecasts, observations and category_edge are expected in absolute units or probabilities consistently. + ``category_edges`` decides how xs.rps interprets forecasts and observations. - - array_like: Category bin edges used to compute the CDFs. Similar to - np.histogram, all but the last (righthand-most) bin include the - left edge and exclude the right edge. The last bin includes both edges. - CDFs based on boolean or logical (True or 1 for event occurance, False or 0 - for non-occurance) observations. - If ``fair==False``, forecasts should be between 0 and 1 without a dimension - ``member_dim`` or boolean / binary containing a member dimension - (probabilities will be internally calculated by ``.mean(member_dim))``. - If ``fair==True``, forecasts must be boolean / binary containing dimension - ``member_dim``. + - np.array (1d): will be internally converted to xr.DataArray or xr.Dataset + and broadcasted to observations. - - xr.Dataset/xr.DataArray: edges of the categories in absolute units provided - as dimension ``category_edges``. These edges must span the full range of the observations and forecasts distribution. Forecasts and observations are expected in absolute units. + - xr.Dataset/xr.DataArray: edges of the categories provided + as dimension ``category_edge`` with optional category labels as ``category_edge`` coordinate. - tuple of xr.Dataset/xr.DataArray: same as xr.Dataset/xr.DataArray, where the first item is taken as ``category_edges`` for observations and the second item for ``category_edges`` for forecasts. - None: expect than observations and forecasts are already CDFs containing - ``category_edges`` dimension. + ``category_edge`` dimension. dim : str or list of str, optional - Dimension over which to compute mean after computing ``rps``. - Defaults to None implying averaging over all dimensions. + Dimension over which to mean after computing ``rps``. This represents a mean + over multiple forecasts-observations pairs. Defaults to None implying averaging + over all dimensions. fair: boolean Apply ensemble member-size adjustment for unbiased, fair metric; - see Ferro (2013). Defaults to False. + see Ferro (2013). If ``fair==True``, forecasts must contain the dimension + ``member_dim``. Defaults to False. weights : xr.DataArray with dimensions from dim, optional Weights for `weighted.mean(dim)`. Defaults to None, such that no weighting is applied. @@ -624,7 +621,16 @@ def rps( _check_identical_xr_types(observations, forecasts) - # different ways of calculating RPS based on category_edges + # different entry point of calculating RPS based on category_edges + if isinstance(category_edges, np.ndarray): + # prepare category_edges as xr object + category_edges = xr.DataArray( + category_edges, + dims="category_edge", + coords={"category_edge": category_edges}, + ) + category_edges = xr.ones_like(observations) * category_edges + if isinstance(category_edges, (xr.Dataset, xr.DataArray)) or isinstance( category_edges, tuple ): @@ -641,75 +647,38 @@ def rps( _check_data_within_edges(forecasts, forecasts_edges) _check_data_within_edges(observations, observations_edges) + def add_eps_to_last_in_dim(category_edges, dim): + """Add 10 eps to last edge to get last bin [ ] instead of [ ) like in xskillscore.core.utils.histogram""" + if isinstance(category_edges, xr.Dataset): + v1 = list(category_edges.data_vars)[0] + dtype = category_edges[v1] + else: + dtype = category_edges.dtype + eps = np.finfo(dtype).eps + category_edges_eps = xr.concat( + [ + category_edges.isel({dim: slice(None, -1)}), + category_edges.isel({dim: [-1]}) + 10 * eps, + ], + dim, + ) + return category_edges_eps + + forecasts_edges = add_eps_to_last_in_dim(forecasts_edges, bin_dim) + observations_edges = add_eps_to_last_in_dim(observations_edges, bin_dim) + # cumulative probs, ignore lowest threshold as below category_edges - Fc = (forecasts < forecasts_edges).mean("member").diff(bin_dim).cumsum(bin_dim) + Fc = ( + (forecasts < forecasts_edges) + .mean(member_dim) + .isel({bin_dim: slice(1, None)}) + ) Oc = ( (observations < observations_edges) .astype("int") .isel({bin_dim: slice(1, None)}) ) - def _check_CDF(cdf): - # CDF <=1 - assert (Fc <= 1.0).all(), print(Fc) - # CDF >=0 - assert (Fc >= 0.0).all(), print(Fc) - # CDF monotonic increasing - assert (Fc.diff("category_edge") >= 0).all() - - _check_CDF(Fc) - _check_CDF(Oc) - - Fc[bin_dim] = _get_category_bounds(forecasts_edges) - Oc[bin_dim] = _get_category_bounds(observations_edges) - - elif isinstance(category_edges, np.ndarray): - # category_edges are 1d array and probability edges - # check monotonic increasing - if np.all(np.diff(category_edges) < 0): - raise ValueError(f"must be monotonic increasing, found {category_edges}") - if category_edges.min() < 0 or category_edges.max() > 1: - raise ValueError(f"must be in [0, 1], found {category_edges}") - # histogram(dim=[]) not allowed therefore add fake member dim - # to apply over when multi-dim observations - if len(observations.dims) == 1: - observations_bins = histogram( - observations, - bins=[category_edges], - bin_names=["category_edge"], - dim=None, - ) - else: - observations_bins = histogram( - observations.expand_dims(member_dim), - bins=[category_edges], - bin_names=["category_edge"], - dim=[member_dim], - ) # .squeeze() - if "category_edge_bin" in observations_bins.dims: - observations_bins = observations_bins.rename( - {"category_edge_bin": "category_edge"} - ) - - forecasts = histogram( - forecasts, - bins=[category_edges], - # bin_names=bin_names, - bin_names=["category_edge"], - dim=[member_dim], - ) - if "category_edge_bin" in forecasts.dims: - forecasts = forecasts.rename({"category_edge_bin": "category_edge"}) - - # normalize f.sum()=1 to make cdf - forecasts = forecasts / forecasts.sum(bin_dim) - - Fc = forecasts.cumsum(bin_dim) - Oc = observations_bins.cumsum(bin_dim) - - Fc[bin_dim] = _get_category_bounds(category_edges) - Oc[bin_dim] = _get_category_bounds(category_edges) - elif category_edges is None: # expect CDFs already as inputs if member_dim in forecasts.dims: forecasts = forecasts.mean(member_dim) @@ -721,11 +690,20 @@ def _check_CDF(cdf): ) # check and maybe rename edges dim - Fc = _check_bin_dim(Fc) - Oc = _check_bin_dim(Oc) - - assert (Fc <= 1.0).all(), print("Fc > 1", Fc) - assert (Oc <= 1.0).all(), print("Oc > 1", Oc) + def _check_is_CDF(cdf): + # CDF <=1 + assert (Fc <= 1.0).all(), print(Fc) + # CDF >=0 + assert (Fc >= 0.0).all(), print(Fc) + # CDF monotonic increasing + assert (Fc.diff("category_edge") >= 0).all() + + _check_is_CDF(Fc) + _check_is_CDF(Oc) + + if category_edges is not None: + Fc[bin_dim] = _get_category_bounds(forecasts_edges) + Oc[bin_dim] = _get_category_bounds(observations_edges) # RPS formulas if fair: @@ -734,17 +712,22 @@ def _check_CDF(cdf): else: res = ((Fc - Oc) ** 2).sum(bin_dim) - # add bin edges as coords - res = res.assign_coords( - {"forecasts_category_edge": ", ".join(_get_category_bounds(Fc[bin_dim].values))} - ) - res = res.assign_coords( - { - "observations_category_edge": ", ".join( - _get_category_bounds(Oc[bin_dim].values) - ) - } - ) + # add category_edge as str into coords + if category_edges is not None: + res = res.assign_coords( + { + "forecasts_category_edge": ", ".join( + _get_category_bounds(forecasts_edges[bin_dim].values) + ) + } + ) + res = res.assign_coords( + { + "observations_category_edge": ", ".join( + _get_category_bounds(observations_edges[bin_dim].values) + ) + } + ) if weights is not None: res = res.weighted(weights) diff --git a/xskillscore/tests/test_probabilistic.py b/xskillscore/tests/test_probabilistic.py index e2b234da..028dadf9 100644 --- a/xskillscore/tests/test_probabilistic.py +++ b/xskillscore/tests/test_probabilistic.py @@ -467,6 +467,99 @@ def test_rps_reduce_dim(o, f_prob, category_edges, dim, fair_bool): assert_only_dim_reduced(dim, actual, o) +def rps_xhist( + observations, + forecasts, + category_edges, + dim=None, + fair=False, + weights=None, + keep_attrs=False, + member_dim="member", +): + """Old way to calculate RPS with xhistogram. + + category_edges : array_like, xr.Dataset, xr.DataArray, None + + - array_like: Category bin edges used to compute the CDFs. Similar to + np.histogram, all but the last (righthand-most) bin include the + left edge and exclude the right edge. The last bin includes both edges. + CDFs based on boolean or logical (True or 1 for event occurance, False or 0 + for non-occurance) observations. + If ``fair==False``, forecasts should be between 0 and 1 without a dimension + ``member_dim`` or boolean / binary containing a member dimension + (probabilities will be internally calculated by ``.mean(member_dim))``. + If ``fair==True``, forecasts must be boolean / binary containing dimension + ``member_dim``.""" + from xskillscore.core.contingency import _get_category_bounds + from xskillscore.core.utils import _keep_nans_masked, histogram + + bin_names = ["category"] + bin_dim = f"{bin_names[0]}_edge" + M = forecasts[member_dim].size + + assert isinstance(category_edges, np.ndarray) + + # histogram(dim=[]) not allowed therefore add fake member dim + # to apply over when multi-dim observations + if len(observations.dims) == 1: + observations_bins = histogram( + observations, + bins=[category_edges], + bin_names=["category_edge"], + dim=None, + ) + else: + observations_bins = histogram( + observations.expand_dims(member_dim), + bins=[category_edges], + bin_names=["category_edge"], + dim=[member_dim], + ) + if "category_edge_bin" in observations_bins.dims: + observations_bins = observations_bins.rename( + {"category_edge_bin": "category_edge"} + ) + + forecasts = histogram( + forecasts, + bins=[category_edges], + bin_names=["category_edge"], + dim=[member_dim], + ) + if "category_edge_bin" in forecasts.dims: + forecasts = forecasts.rename({"category_edge_bin": "category_edge"}) + + # normalize f.sum()=1 to make cdf + forecasts = forecasts / forecasts.sum(bin_dim) + + Fc = forecasts.cumsum(bin_dim) + Oc = observations_bins.cumsum(bin_dim) + + # RPS formulas + if fair: + Ec = Fc * M + res = ((Ec / M - Oc) ** 2 - Ec * (M - Ec) / (M ** 2 * (M - 1))).sum(bin_dim) + else: + res = ((Fc - Oc) ** 2).sum(bin_dim) + + if weights is not None: + res = res.weighted(weights) + + res = res.mean(dim, keep_attrs=keep_attrs) + # add bin edges as coords + res = res.assign_coords( + {"forecasts_category_edge": ", ".join(_get_category_bounds(category_edges))} + ) + res = res.assign_coords( + {"observations_category_edge": ", ".join(_get_category_bounds(category_edges))} + ) + + # keep nans and prevent 0 for all nan grids + res = _keep_nans_masked(observations, res, dim, ignore=["category_edge"]) + return res + + def test_rps_wilks_example(): """Test with values from Wilks, D. S. (2006). Statistical methods in the atmospheric sciences (2nd ed, Vol. 91). Amsterdam ; Boston: Academic Press. p.301. @@ -481,8 +574,8 @@ def test_rps_wilks_example(): F2 = xr.DataArray( [0] * 2 + [0.1] * 3 + [0.3] * 5, dims="member" ) # .expand_dims('time') - np.testing.assert_allclose(rps(Obs, F1, category_edges), 0.73) - np.testing.assert_allclose(rps(Obs, F2, category_edges), 0.89) + np.testing.assert_allclose(rps_xhist(Obs, F1, category_edges), 0.73) + np.testing.assert_allclose(rps_xhist(Obs, F2, category_edges), 0.89) # xr way with xr.DataArray category_edges xr_category_edges = xr.DataArray( category_edges, dims="category_edge", coords={"category_edge": category_edges} @@ -492,8 +585,8 @@ def test_rps_wilks_example(): # second example Obs = xr.DataArray([0.3]) # larger than 0.25 - np.testing.assert_allclose(rps(Obs, F1, category_edges), 0.53) - np.testing.assert_allclose(rps(Obs, F2, category_edges), 0.29) + np.testing.assert_allclose(rps_xhist(Obs, F1, category_edges), 0.53) + np.testing.assert_allclose(rps_xhist(Obs, F2, category_edges), 0.29) # xr way with xr.DataArray category_edges assert_allclose(rps(Obs, F1, category_edges), rps(Obs, F1, xr_category_edges)) assert_allclose(rps(Obs, F2, category_edges), rps(Obs, F2, xr_category_edges)) @@ -597,9 +690,11 @@ def test_rps_category_edges_tuple(o, f_prob, fair_bool): @pytest.mark.parametrize("fair_bool", [True, False]) def test_rps_category_edges_None(o, f_prob, fair_bool): """Test rps with category_edges as None expecting o and f_prob are already CDFs.""" - edges = xr.DataArray([0, 0.2, 0.4, 0.6, 0.8, 1.0], dims="category_edge") - o_c = o > edges # CDF - f_prob_c = f_prob > edges # CDF + e = [0, 0.2, 0.4, 0.6, 0.8, 1.0] + bin_dim = "category_edge" + edges = xr.DataArray(e, dims=bin_dim, coords={bin_dim: e}) + o_c = o < edges # CDF + f_prob_c = f_prob < edges # CDF actual = rps(o_c, f_prob_c, dim="time", fair=fair_bool, category_edges=None) assert set(["lon", "lat"]) == set(actual.dims) assert "quantile" not in actual.dims @@ -639,19 +734,21 @@ def test_rps_new_identical_old_xhistogram(o, f_prob, fair_bool): coords={"category_edge": category_edges_np}, ) dim = "time" - f_prob = f_prob.isel(lon=2, time=slice(None, 3), drop=True) - o = o.isel(lon=2, time=slice(None, 3), drop=True) - print("member", f_prob.member.size) - print("f_prob", f_prob) - print("o", o) - print("category_edges_xr") actual = rps(o, f_prob, dim=dim, category_edges=category_edges_xr) - print("\n\n category_edges_np") - expected = rps(o, f_prob, dim=dim, category_edges=category_edges_np) - print("actual", actual) - print("expected", expected) + expected = rps_xhist(o, f_prob, dim=dim, category_edges=category_edges_np) + print(actual.coords["forecasts_category_edge"].values) + print(expected.coords["forecasts_category_edge"].values) assert_allclose(actual.rename("histogram_category_edge"), expected) - # assert False + + +def test_rps_last_edge_included(o, f_prob): + category_edges_np = np.array([0, 0.2, 0.4, 0.6, 0.8, 1.0]) + o = xr.ones_like(o) + f_prob = xr.ones_like(f_prob) + res_actual = rps(o, f_prob, dim="time", category_edges=category_edges_np) + print(res_actual) + print(res_actual.coords["forecasts_category_edge"]) + assert (res_actual == 0).all() @pytest.mark.parametrize( From 36e4eebebf957ae2aed90311736a7102cd98fa98 Mon Sep 17 00:00:00 2001 From: AS Date: Mon, 1 Mar 2021 17:24:24 +0100 Subject: [PATCH 15/39] fix docstring --- xskillscore/core/probabilistic.py | 53 ++++++++++++++++++------- xskillscore/tests/test_probabilistic.py | 38 ++++++++++++++++++ 2 files changed, 77 insertions(+), 14 deletions(-) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index 47970d11..110455c4 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -524,7 +524,6 @@ def rps( weights=None, keep_attrs=False, member_dim="member", - # skipna=False, ): """Calculate Ranked Probability Score. @@ -585,23 +584,39 @@ def rps( Returns ------- xarray.Dataset or xarray.DataArray: - ranked probability score + ranked probability score with coords ``forecasts_category_edge`` and ``observations_category_edge`` as str + Examples -------- - >>> observations = xr.DataArray(np.random.normal(size=(3,3)), + >>> observations = xr.DataArray(np.random.random(size=(3,3)), ... coords=[('x', np.arange(3)), ... ('y', np.arange(3))]) - >>> forecasts = xr.DataArray(np.random.normal(size=(3,3,3)), + >>> forecasts = xr.DataArray(np.random.random(size=(3,3,3)), ... coords=[('x', np.arange(3)), ... ('y', np.arange(3)), ... ('member', np.arange(3))]) - >>> category_edges = np.array([.2, .5, .8]) - >>> rps(observations > 0.5, (forecasts > 0.5).mean('member'), category_edges) - - array([1. , 1. , 0.33333333]) + >>> category_edges = np.array([.0, .5, 1.]) + >>> xs.rps(observations, forecasts, category_edges, dim='x') + + array([0.85185185, 0.59259259, 0.37037037]) Coordinates: - * y (y) int64 0 1 2 + * y (y) int64 0 1 2 + forecasts_category_edge >> category_edges = xr.concat([ + ... xr.DataArray(0).expand_dims('category_edge').assign_coords(category_edge=[0]), + ... observations.quantile(q=[.33, .66]).rename({'quantile':'category_edge'}), + ... xr.DataArray(1).expand_dims('category_edge').assign_coords(category_edge=[1]) + ... ],'category_edge') + >>> xs.rps(observations, forecasts, category_edges, dim='x') + + array([1.18518519, 0.85185185, 0.40740741]) + Coordinates: + * y (y) int64 0 1 2 + forecasts_category_edge Date: Mon, 1 Mar 2021 17:28:46 +0100 Subject: [PATCH 16/39] utils --- xskillscore/core/utils.py | 29 ++--------------------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/xskillscore/core/utils.py b/xskillscore/core/utils.py index 01b95cf6..f50137e4 100644 --- a/xskillscore/core/utils.py +++ b/xskillscore/core/utils.py @@ -193,44 +193,17 @@ def _check_identical_xr_types(a, b): def _keep_nans_masked(ds_before, ds_after, dim=None, ignore=None): """Preserve all NaNs from ds_before for ds_after over while ignoring some dimensions optionally.""" - print( - "ds_before.dims =", - ds_before.dims, - "ds_after.dims", - ds_after.dims, - " dim =", - dim, - ) if dim is None: dim = list(ds_before.dims) elif isinstance(dim, str): dim = [dim] - print("dim =", dim) - if ignore is None: ignore = [] elif isinstance(ignore, str): ignore = list(ignore) - - # dim = set(dim) - set(ignore) - - # all_dim = ds_before.dims #[d for d in dim if d in ds_before.dims] all_dim = set(dim) ^ set(ignore) - print("all_dim", all_dim) all_dim = [d for d in all_dim if d in ds_before.dims] - print("all_dim", all_dim) - mask = ds_before.isnull().all(all_dim) - print("mask", mask) - if False: - for d in dim: - assert d not in mask - if ignore is not None: - overlap_dims = set(mask.dims) & set(ignore) - print("overlap_dims", overlap_dims) - if len(overlap_dims) > 0: - mask = mask.mean(overlap_dims) - print("mask.dims", mask.dims) ds_after = ds_after.where(~mask.astype("bool"), other=np.nan) for d in dim: assert d not in ds_after.dims @@ -243,6 +216,8 @@ def _keep_nans_masked(ds_before, ds_after, dim=None, ignore=None): def quantile_edges_pad_lower_upper( ds, edges, dim, lower=0, upper=1, category_edge_dim="category_edge" ): + """Convenience function to get category_edges for xs.rps based on quantiles with + lower as first category_edge and upper as last category_edge.""" assert isinstance(dim, list) if isinstance(edges, list): edges = np.array(edges) From f43c9c914d585103fbbaf4f829a0345370c08d43 Mon Sep 17 00:00:00 2001 From: AS Date: Mon, 1 Mar 2021 17:45:55 +0100 Subject: [PATCH 17/39] cleanup --- xskillscore/core/probabilistic.py | 87 +++++++++++++------------ xskillscore/tests/test_probabilistic.py | 28 +------- 2 files changed, 49 insertions(+), 66 deletions(-) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index 110455c4..b78c21e4 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -515,6 +515,44 @@ def _check(forecasts, forecasts_edges): from .contingency import _get_category_bounds +def _add_eps_to_last_in_dim(category_edges, dim): + """Add 10 eps to last edge to get last bin [ ] instead of [ ) like in xskillscore.core.utils.histogram""" + if isinstance(category_edges, xr.Dataset): + v1 = list(category_edges.data_vars)[0] + dtype = category_edges[v1] + else: + dtype = category_edges.dtype + eps = np.finfo(dtype).eps + category_edges_eps = xr.concat( + [ + category_edges.isel({dim: slice(None, -1)}), + category_edges.isel({dim: [-1]}) + 10 * eps, + ], + dim, + ) + return category_edges_eps + + +def _check_is_CDF(cdf): + """Check basic characteristics of a cumulative distribution function.""" + + def func(cdf): + # CDF <=1 + if not (cdf <= 1.0).all(): + raise ValueError(f"Found CDF > 1, max = {cdf.max()}") + # CDF >=0 + if not (cdf >= 0.0).all(): + raise ValueError(f"Found CDF < 0, min = {cdf.min()}") + # CDF monotonic increasing + if not (cdf.astype("float").diff("category_edge") >= 0).all(): + raise ValueError("Found CDF not monotonic increasing") + + if isinstance(cdf, xr.Dataset): + cdf.map(func) + elif isinstance(cdf, xr.DataArray): + func(cdf) + + def rps( observations, forecasts, @@ -605,6 +643,9 @@ def rps( forecasts_category_edge >> category_edges = xr.concat([ ... xr.DataArray(0).expand_dims('category_edge').assign_coords(category_edge=[0]), ... observations.quantile(q=[.33, .66]).rename({'quantile':'category_edge'}), @@ -663,25 +704,8 @@ def rps( _check_data_within_edges(forecasts, forecasts_edges) _check_data_within_edges(observations, observations_edges) - def add_eps_to_last_in_dim(category_edges, dim): - """Add 10 eps to last edge to get last bin [ ] instead of [ ) like in xskillscore.core.utils.histogram""" - if isinstance(category_edges, xr.Dataset): - v1 = list(category_edges.data_vars)[0] - dtype = category_edges[v1] - else: - dtype = category_edges.dtype - eps = np.finfo(dtype).eps - category_edges_eps = xr.concat( - [ - category_edges.isel({dim: slice(None, -1)}), - category_edges.isel({dim: [-1]}) + 10 * eps, - ], - dim, - ) - return category_edges_eps - - forecasts_edges = add_eps_to_last_in_dim(forecasts_edges, bin_dim) - observations_edges = add_eps_to_last_in_dim(observations_edges, bin_dim) + forecasts_edges = _add_eps_to_last_in_dim(forecasts_edges, bin_dim) + observations_edges = _add_eps_to_last_in_dim(observations_edges, bin_dim) # cumulative probs, ignore lowest threshold as below category_edges Fc = ( @@ -705,27 +729,14 @@ def add_eps_to_last_in_dim(category_edges, dim): f"category_edges must be xr.DataArray, xr.Dataset, tuple of xr.objects, None or array-like, found {type(category_edges)}" ) - # check and maybe rename edges dim - def _check_is_CDF(cdf): - # CDF <=1 - assert (Fc <= 1.0).all(), print(Fc) - # CDF >=0 - assert (Fc >= 0.0).all(), print(Fc) - # CDF monotonic increasing - assert (Fc.diff("category_edge") >= 0).all() - _check_is_CDF(Fc) _check_is_CDF(Oc) - # if category_edges is not None: - # Fc[bin_dim] = _get_category_bounds(forecasts_edges) - # Oc[bin_dim] = _get_category_bounds(observations_edges) - # RPS formulas - if fair: + if fair: # for ensemble member adjustment Ferro 2013 Ec = Fc * M res = ((Ec / M - Oc) ** 2 - Ec * (M - Ec) / (M ** 2 * (M - 1))).sum(bin_dim) - else: + else: # normal formula res = ((Fc - Oc) ** 2).sum(bin_dim) # add category_edge as str into coords @@ -744,17 +755,13 @@ def _check_is_CDF(cdf): ) } ) - if weights is not None: res = res.weighted(weights) - + # combine many forecasts-observations pairs res = res.mean(dim) - # keep nans and prevent 0 for all nan grids res = _keep_nans_masked(observations, res, dim, ignore=["category_edge"]) - - if keep_attrs: - print(type(res.attrs), type(res)) + if keep_attrs: # attach by hand res.attrs.update(observations.attrs) res.attrs.update(forecasts.attrs) if isinstance(res, xr.Dataset): diff --git a/xskillscore/tests/test_probabilistic.py b/xskillscore/tests/test_probabilistic.py index 07d6fd72..7cc621ca 100644 --- a/xskillscore/tests/test_probabilistic.py +++ b/xskillscore/tests/test_probabilistic.py @@ -467,22 +467,6 @@ def test_rps_reduce_dim(o, f_prob, category_edges, dim, fair_bool): assert_only_dim_reduced(dim, actual, o) -@pytest.mark.parametrize("fair_bool", [True, False]) -def test_rps_skipna(o, f_prob, category_edges, fair_bool): - """Test that rps reduced dim and works for (chunked) ds and da""" - dim = "time" - o[0, 0, 0] = np.nan - skipna_True = rps( - o, f_prob, category_edges=category_edges, dim=dim, fair=fair_bool, skipna=True - ) - skipna_False = rps( - o, f_prob, category_edges=category_edges, dim=dim, fair=fair_bool, skipna=True - ) - print(skipna_True) - print(skipna_True - skipna_False) - assert False # (skipna_True - skipna_False).notnull().any() - - @pytest.mark.parametrize("chunk_bool", [True, False]) @pytest.mark.parametrize("input_type", ["Dataset", "multidim Dataset", "DataArray"]) @pytest.mark.parametrize("keep_attrs", [True, False]) @@ -751,14 +735,10 @@ def test_rps_keeps_masked(o, f_prob, fair_bool, category_edges): """Test rps keeps NaNs.""" o = o.where(o.lat > 1) f_prob = f_prob.where(f_prob.lat > 1) - print("o", o) - print("f_prob", f_prob) actual = rps(o, f_prob, dim="time", category_edges=category_edges) assert set(["lon", "lat"]) == set(actual.dims) - assert actual.isel(lat=[0, 1]).isnull().all(), print(actual.isel(lat=[0, 1])) - assert actual.isel(lat=slice(2, None)).notnull().all(), print( - actual.isel(lat=slice(2, None)) - ) + assert actual.isel(lat=[0, 1]).isnull().all() + assert actual.isel(lat=slice(2, None)).notnull().all() @pytest.mark.parametrize("fair_bool", [True, False], ids=["bool=fair", "fair=False"]) @@ -774,8 +754,6 @@ def test_rps_new_identical_old_xhistogram(o, f_prob, fair_bool): dim = "time" actual = rps(o, f_prob, dim=dim, category_edges=category_edges_xr) expected = rps_xhist(o, f_prob, dim=dim, category_edges=category_edges_np) - print(actual.coords["forecasts_category_edge"].values) - print(expected.coords["forecasts_category_edge"].values) assert_allclose(actual.rename("histogram_category_edge"), expected) @@ -784,8 +762,6 @@ def test_rps_last_edge_included(o, f_prob): o = xr.ones_like(o) f_prob = xr.ones_like(f_prob) res_actual = rps(o, f_prob, dim="time", category_edges=category_edges_np) - print(res_actual) - print(res_actual.coords["forecasts_category_edge"]) assert (res_actual == 0).all() From 9d5443babbeea4b6ffeb7ca0a8e2b52950072264 Mon Sep 17 00:00:00 2001 From: AS Date: Mon, 1 Mar 2021 17:57:04 +0100 Subject: [PATCH 18/39] rtd --- xskillscore/core/probabilistic.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index b78c21e4..6273f9fa 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -585,15 +585,19 @@ def rps( The forecast of the event. Further requirements are specified based on ``category_edges``. category_edges : array_like, xr.Dataset, xr.DataArray, None - Category edges used to compute the CDFs. Similar to np.histogram, all but the last (righthand-most) bin include the left edge and exclude the right edge. The last bin includes both edges. - These ``category_edge`` must span the full range of the observations and forecasts distribution. Forecasts, observations and category_edge are expected in absolute units or probabilities consistently. + Category edges used to compute the CDFs. Similar to np.histogram, all but the + last (righthand-most) bin include the left edge and exclude the right edge. The + last bin includes both edges. + These ``category_edge`` must span the full range of the observations and + forecasts distribution. Forecasts, observations and category_edge are expected + in absolute units or probabilities consistently. ``category_edges`` decides how xs.rps interprets forecasts and observations. - - np.array (1d): will be internally converted to xr.DataArray or xr.Dataset - and broadcasted to observations. + - np.array (1d): will be internally converted and broadcasted to observations. - xr.Dataset/xr.DataArray: edges of the categories provided - as dimension ``category_edge`` with optional category labels as ``category_edge`` coordinate. + as dimension ``category_edge`` with optional category labels as + ``category_edge`` coordinate. - tuple of xr.Dataset/xr.DataArray: same as xr.Dataset/xr.DataArray, where the first item is taken as ``category_edges`` for observations and the second item @@ -622,7 +626,8 @@ def rps( Returns ------- xarray.Dataset or xarray.DataArray: - ranked probability score with coords ``forecasts_category_edge`` and ``observations_category_edge`` as str + ranked probability score with coords ``forecasts_category_edge`` and + ``observations_category_edge`` as str Examples @@ -650,7 +655,7 @@ def rps( ... xr.DataArray(0).expand_dims('category_edge').assign_coords(category_edge=[0]), ... observations.quantile(q=[.33, .66]).rename({'quantile':'category_edge'}), ... xr.DataArray(1).expand_dims('category_edge').assign_coords(category_edge=[1]) - ... ],'category_edge') + ... ],'category_edge') >>> xs.rps(observations, forecasts, category_edges, dim='x') array([1.18518519, 0.85185185, 0.40740741]) From e8a5d14af4149bb752aac5f4a0017c42c4fce2ce Mon Sep 17 00:00:00 2001 From: AS Date: Mon, 1 Mar 2021 19:52:59 +0100 Subject: [PATCH 19/39] rtd --- xskillscore/core/probabilistic.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index 6273f9fa..e8717641 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -648,9 +648,11 @@ def rps( forecasts_category_edge >> category_edges = xr.concat([ ... xr.DataArray(0).expand_dims('category_edge').assign_coords(category_edge=[0]), ... observations.quantile(q=[.33, .66]).rename({'quantile':'category_edge'}), From ed12fad4df9c62d4abcc32280c7c3316f7831881 Mon Sep 17 00:00:00 2001 From: AS Date: Tue, 2 Mar 2021 15:47:20 +0100 Subject: [PATCH 20/39] suggestions from code review --- xskillscore/core/probabilistic.py | 55 +++++++++++++------------ xskillscore/tests/test_probabilistic.py | 12 +++++- 2 files changed, 39 insertions(+), 28 deletions(-) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index e8717641..5075a1d7 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -4,7 +4,7 @@ import properscoring import xarray as xr -from .contingency import Contingency +from .contingency import Contingency, _get_category_bounds from .utils import ( _add_as_coord, _bool_to_int, @@ -487,34 +487,35 @@ def _check_bin_dim(ds): def _check_data_within_edges(forecasts, forecasts_edges): - """Check that forecasts_edges has category_edge dimension and forecasts_edges span range of forecasts (per variable if xr.Dataset).""" + """Check that forecasts_edges has category_edge dimension and forecasts_edges span + range of forecasts (per variable if xr.Dataset), otherwise ValueError.""" if "category_edge" not in forecasts_edges.dims: raise ValueError( - f"Expect to find category_edge in forecast_edges, found {forecasts_edges.dims}" + "Expect to find category_edge in forecast_edges, found " + f"{forecasts_edges.dims}" ) - def _check(forecasts, forecasts_edges): + def _helper_check_data_within_edges(forecasts, forecasts_edges): if forecasts.min() < forecasts_edges.min(): raise ValueError( - f"found forecasts outside forecast_edges, found forecasts_edges.min() = \n{forecasts_edges.min()}\n and forecasts.min() = \n {forecasts.min()}" + "found forecasts outside forecast_edges, found forecasts_edges.min() = " + f"{forecasts_edges.min()} and forecasts.min() = {forecasts.min()}" ) if forecasts.max() > forecasts_edges.max(): raise ValueError( - f"found forecasts outside forecast_edges, found forecasts_edges.max() = \n{forecasts_edges.max()}\n and forecasts.max() = \n {forecasts.max()}" + "found forecasts outside forecast_edges, found forecasts_edges.max() = " + f"{forecasts_edges.max()} and forecasts.max() = {forecasts.max()}" ) if isinstance(forecasts, xr.Dataset): for v in forecasts.data_vars: - _check(forecasts[v], forecasts_edges[v]) + _helper_check_data_within_edges(forecasts[v], forecasts_edges[v]) elif isinstance(forecasts, xr.DataArray): - _check(forecasts, forecasts_edges) + _helper_check_data_within_edges(forecasts, forecasts_edges) else: raise ValueError("only defined for xr.DataArrays and xr.Datasets") -from .contingency import _get_category_bounds - - def _add_eps_to_last_in_dim(category_edges, dim): """Add 10 eps to last edge to get last bin [ ] instead of [ ) like in xskillscore.core.utils.histogram""" if isinstance(category_edges, xr.Dataset): @@ -522,21 +523,20 @@ def _add_eps_to_last_in_dim(category_edges, dim): dtype = category_edges[v1] else: dtype = category_edges.dtype - eps = np.finfo(dtype).eps category_edges_eps = xr.concat( [ category_edges.isel({dim: slice(None, -1)}), - category_edges.isel({dim: [-1]}) + 10 * eps, + category_edges.isel({dim: [-1]}) + 10 * np.finfo(dtype).eps, ], dim, ) return category_edges_eps -def _check_is_CDF(cdf): +def _raise_value_error_cdf(cdf): """Check basic characteristics of a cumulative distribution function.""" - def func(cdf): + def helper_raise_value_error_cdf(cdf): # CDF <=1 if not (cdf <= 1.0).all(): raise ValueError(f"Found CDF > 1, max = {cdf.max()}") @@ -548,9 +548,9 @@ def func(cdf): raise ValueError("Found CDF not monotonic increasing") if isinstance(cdf, xr.Dataset): - cdf.map(func) + cdf.map(helper_raise_value_error_cdf) elif isinstance(cdf, xr.DataArray): - func(cdf) + helper_raise_value_error_cdf(cdf) def rps( @@ -632,21 +632,21 @@ def rps( Examples -------- - >>> observations = xr.DataArray(np.random.random(size=(3,3)), + >>> observations = xr.DataArray(np.random.random(size=(3, 3)), ... coords=[('x', np.arange(3)), ... ('y', np.arange(3))]) - >>> forecasts = xr.DataArray(np.random.random(size=(3,3,3)), + >>> forecasts = xr.DataArray(np.random.random(size=(3, 3, 3)), ... coords=[('x', np.arange(3)), ... ('y', np.arange(3)), ... ('member', np.arange(3))]) >>> category_edges = np.array([.0, .5, 1.]) >>> xs.rps(observations, forecasts, category_edges, dim='x') - array([0.85185185, 0.59259259, 0.37037037]) + array([0.14814815, 0.7037037 , 1.51851852]) Coordinates: * y (y) int64 0 1 2 - forecasts_category_edge Date: Tue, 2 Mar 2021 16:07:29 +0100 Subject: [PATCH 21/39] allow tuple of np.ndarray also, refactor --- xskillscore/core/probabilistic.py | 52 +++++++++++++++++++------------ 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index 5075a1d7..fd410f11 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -599,7 +599,7 @@ def rps( as dimension ``category_edge`` with optional category labels as ``category_edge`` coordinate. - - tuple of xr.Dataset/xr.DataArray: same as xr.Dataset/xr.DataArray, where the + - tuple of np.array/xr.Dataset/xr.DataArray: same as above, where the first item is taken as ``category_edges`` for observations and the second item for ``category_edges`` for forecasts. @@ -685,28 +685,40 @@ def rps( _check_identical_xr_types(observations, forecasts) # different entry point of calculating RPS based on category_edges - if isinstance(category_edges, np.ndarray): - # prepare category_edges as xr object - category_edges = xr.DataArray( - category_edges, - dims="category_edge", - coords={"category_edge": category_edges}, - ) - category_edges = xr.ones_like(observations) * category_edges - - if isinstance(category_edges, (xr.Dataset, xr.DataArray)) or isinstance( - category_edges, tuple - ): - if isinstance( - category_edges, tuple - ): # edges tuple of two: use for obs and forecast edges separately + # category_edges tuple of two: use for obs and forecast category_edges separately + if isinstance(category_edges, (tuple, np.ndarray, xr.DataArray, xr.Dataset)): + if isinstance(category_edges, tuple): + assert isinstance(category_edges[0], type(category_edges[1])) observations_edges, forecasts_edges = category_edges - _check_identical_xr_types(forecasts_edges, forecasts) - _check_identical_xr_types(observations_edges, forecasts) - else: # edges only given once, so use for both obs and forecasts - _check_identical_xr_types(category_edges, forecasts) + else: # category_edges only given once, so use for both obs and forecasts observations_edges, forecasts_edges = category_edges, category_edges + if isinstance(observations_edges, np.ndarray): + # convert category_edges as xr object + observations_edges = xr.DataArray( + observations_edges, + dims="category_edge", + coords={"category_edge": observations_edges}, + ) + observations_edges = xr.ones_like(observations) * observations_edges + + forecasts_edges = xr.DataArray( + forecasts_edges, + dims="category_edge", + coords={"category_edge": forecasts_edges}, + ) + forecasts_edges = ( + xr.ones_like( + forecasts + if member_dim not in forecasts.dims + else forecasts.isel({member_dim: 0}, drop=True) + ) + * forecasts_edges + ) + + _check_identical_xr_types(forecasts_edges, forecasts) + _check_identical_xr_types(observations_edges, forecasts) + _check_data_within_edges(forecasts, forecasts_edges) _check_data_within_edges(observations, observations_edges) From a012fe8e11c5e941c651dda957601c81c126797d Mon Sep 17 00:00:00 2001 From: AS Date: Tue, 2 Mar 2021 16:10:21 +0100 Subject: [PATCH 22/39] docstr --- xskillscore/core/probabilistic.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index fd410f11..dcf860e2 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -597,7 +597,8 @@ def rps( - xr.Dataset/xr.DataArray: edges of the categories provided as dimension ``category_edge`` with optional category labels as - ``category_edge`` coordinate. + ``category_edge`` coordinate. Use xr.Dataset/xr.DataArray if edges + multi-dimensional and vary across dimensions. - tuple of np.array/xr.Dataset/xr.DataArray: same as above, where the first item is taken as ``category_edges`` for observations and the second item From 243f1a6e07ee2648d4f8b48b33009413edceaf8f Mon Sep 17 00:00:00 2001 From: Aaron Spring Date: Thu, 4 Mar 2021 09:57:44 +0100 Subject: [PATCH 23/39] rm add_eps_to_last_edge as last category is unlimited here --- xskillscore/core/probabilistic.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index dcf860e2..10ab553a 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -516,23 +516,6 @@ def _helper_check_data_within_edges(forecasts, forecasts_edges): raise ValueError("only defined for xr.DataArrays and xr.Datasets") -def _add_eps_to_last_in_dim(category_edges, dim): - """Add 10 eps to last edge to get last bin [ ] instead of [ ) like in xskillscore.core.utils.histogram""" - if isinstance(category_edges, xr.Dataset): - v1 = list(category_edges.data_vars)[0] - dtype = category_edges[v1] - else: - dtype = category_edges.dtype - category_edges_eps = xr.concat( - [ - category_edges.isel({dim: slice(None, -1)}), - category_edges.isel({dim: [-1]}) + 10 * np.finfo(dtype).eps, - ], - dim, - ) - return category_edges_eps - - def _raise_value_error_cdf(cdf): """Check basic characteristics of a cumulative distribution function.""" @@ -723,10 +706,6 @@ def rps( _check_data_within_edges(forecasts, forecasts_edges) _check_data_within_edges(observations, observations_edges) - # make last category_bin to include last edge [ ] - forecasts_edges = _add_eps_to_last_in_dim(forecasts_edges, bin_dim) - observations_edges = _add_eps_to_last_in_dim(observations_edges, bin_dim) - # cumulative probs, ignore lowest threshold as below category_edges Fc = ( (forecasts < forecasts_edges) From 500ee62bd2911a50cbe12255d3ed25a542a1902e Mon Sep 17 00:00:00 2001 From: Aaron Spring Date: Sat, 6 Mar 2021 17:05:50 +0100 Subject: [PATCH 24/39] Update probabilistic.py --- xskillscore/core/probabilistic.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index 10ab553a..b4cd52b8 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -565,7 +565,7 @@ def rps( The observations of the event. Further requirements are specified based on ``category_edges``. forecasts : xarray.Dataset or xarray.DataArray - The forecast of the event. + The forecast of the event with dimension specified by ``member_dim``. Further requirements are specified based on ``category_edges``. category_edges : array_like, xr.Dataset, xr.DataArray, None Category edges used to compute the CDFs. Similar to np.histogram, all but the @@ -595,9 +595,7 @@ def rps( over multiple forecasts-observations pairs. Defaults to None implying averaging over all dimensions. fair: boolean - Apply ensemble member-size adjustment for unbiased, fair metric; - see Ferro (2013). If ``fair==True``, forecasts must contain the dimension - ``member_dim``. Defaults to False. + Apply ensemble member-size adjustment for unbiased, fair metric; see Ferro (2013). weights : xr.DataArray with dimensions from dim, optional Weights for `weighted.mean(dim)`. Defaults to None, such that no weighting is applied. @@ -661,6 +659,9 @@ def rps( """ bin_dim = "category_edge" + if member_dim not in forecasts.dims: + raise ValueError(f"Expect to find {member_dim} in forecasts dimensions, found" + f"{forecasts.dims}.") if fair: M = forecasts[member_dim].size @@ -707,6 +708,8 @@ def rps( _check_data_within_edges(observations, observations_edges) # cumulative probs, ignore lowest threshold as below category_edges + # ignores the right-most edge. The effective right-most edge is np.inf. + # therefore the CDFs Fc and Oc both reach 1 for the right-most edge. Fc = ( (forecasts < forecasts_edges) .mean(member_dim) From 371898e45d3f176f360f8e36a10fa1273ec353de Mon Sep 17 00:00:00 2001 From: Aaron Spring Date: Sat, 6 Mar 2021 20:54:19 +0100 Subject: [PATCH 25/39] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b37423d8..97d42707 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ bottleneck cftime -dask +dask=2021.2 numba>=0.52 numpy properscoring From 47fd8716b413e931c759a9063150a2e47f031111 Mon Sep 17 00:00:00 2001 From: Aaron Spring Date: Sat, 6 Mar 2021 20:57:03 +0100 Subject: [PATCH 26/39] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 97d42707..d624c696 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ bottleneck cftime -dask=2021.2 +dask=2021.02 numba>=0.52 numpy properscoring From 04e8bf562e6399df62238d475c1fed61a04358de Mon Sep 17 00:00:00 2001 From: Aaron Spring Date: Sat, 6 Mar 2021 21:07:33 +0100 Subject: [PATCH 27/39] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d624c696..767d9dd4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ bottleneck cftime -dask=2021.02 +dask=2021.02.0 numba>=0.52 numpy properscoring From 2256ac2a43870d1eaa2ed4dbdb6107e88ff82adf Mon Sep 17 00:00:00 2001 From: Aaron Spring Date: Sat, 6 Mar 2021 21:11:45 +0100 Subject: [PATCH 28/39] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 767d9dd4..43df7bcb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ bottleneck cftime -dask=2021.02.0 +dask==2021.02.0 numba>=0.52 numpy properscoring From 022bbdfa3432e12646491dbd3fa95b24ed0e7d99 Mon Sep 17 00:00:00 2001 From: Aaron Spring Date: Sat, 6 Mar 2021 21:16:41 +0100 Subject: [PATCH 29/39] Update probabilistic.py --- xskillscore/core/probabilistic.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index b4cd52b8..cdc48f00 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -660,8 +660,9 @@ def rps( """ bin_dim = "category_edge" if member_dim not in forecasts.dims: - raise ValueError(f"Expect to find {member_dim} in forecasts dimensions, found" - f"{forecasts.dims}.") + raise ValueError( + f"Expect to find {member_dim} in forecasts dimensions, found" + f"{forecasts.dims}.") if fair: M = forecasts[member_dim].size From b875ddf9f0cd567bbac959574bae13e70b4e2b26 Mon Sep 17 00:00:00 2001 From: Aaron Spring Date: Sat, 6 Mar 2021 21:19:43 +0100 Subject: [PATCH 30/39] Update probabilistic.py --- xskillscore/core/probabilistic.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index cdc48f00..7f70ef82 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -662,7 +662,8 @@ def rps( if member_dim not in forecasts.dims: raise ValueError( f"Expect to find {member_dim} in forecasts dimensions, found" - f"{forecasts.dims}.") + f"{forecasts.dims}." + ) if fair: M = forecasts[member_dim].size From 3dd943d4cfe055258a4fb644af0f2dc0208c4b2b Mon Sep 17 00:00:00 2001 From: AS Date: Tue, 9 Mar 2021 14:58:40 +0100 Subject: [PATCH 31/39] set +/- np.inf as category label, less checks --- xskillscore/core/probabilistic.py | 124 ++++++------------------ xskillscore/core/utils.py | 21 ---- xskillscore/tests/test_probabilistic.py | 39 ++++---- 3 files changed, 49 insertions(+), 135 deletions(-) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index 7f70ef82..86248386 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -478,64 +478,6 @@ def threshold_brier_score( return res.mean(dim, keep_attrs=keep_attrs) -def _check_bin_dim(ds): - """Assert that bin_dim is in ds. Try to guess and rename edges dimension.""" - bin_dim = "category_edge" - if bin_dim not in ds.dims: - raise ValueError(f"require {bin_dim} dimension, found {ds.dims}") - return ds - - -def _check_data_within_edges(forecasts, forecasts_edges): - """Check that forecasts_edges has category_edge dimension and forecasts_edges span - range of forecasts (per variable if xr.Dataset), otherwise ValueError.""" - if "category_edge" not in forecasts_edges.dims: - raise ValueError( - "Expect to find category_edge in forecast_edges, found " - f"{forecasts_edges.dims}" - ) - - def _helper_check_data_within_edges(forecasts, forecasts_edges): - if forecasts.min() < forecasts_edges.min(): - raise ValueError( - "found forecasts outside forecast_edges, found forecasts_edges.min() = " - f"{forecasts_edges.min()} and forecasts.min() = {forecasts.min()}" - ) - if forecasts.max() > forecasts_edges.max(): - raise ValueError( - "found forecasts outside forecast_edges, found forecasts_edges.max() = " - f"{forecasts_edges.max()} and forecasts.max() = {forecasts.max()}" - ) - - if isinstance(forecasts, xr.Dataset): - for v in forecasts.data_vars: - _helper_check_data_within_edges(forecasts[v], forecasts_edges[v]) - elif isinstance(forecasts, xr.DataArray): - _helper_check_data_within_edges(forecasts, forecasts_edges) - else: - raise ValueError("only defined for xr.DataArrays and xr.Datasets") - - -def _raise_value_error_cdf(cdf): - """Check basic characteristics of a cumulative distribution function.""" - - def helper_raise_value_error_cdf(cdf): - # CDF <=1 - if not (cdf <= 1.0).all(): - raise ValueError(f"Found CDF > 1, max = {cdf.max()}") - # CDF >=0 - if not (cdf >= 0.0).all(): - raise ValueError(f"Found CDF < 0, min = {cdf.min()}") - # CDF monotonic increasing - if not (cdf.astype("float").diff("category_edge") >= 0).all(): - raise ValueError("Found CDF not monotonic increasing") - - if isinstance(cdf, xr.Dataset): - cdf.map(helper_raise_value_error_cdf) - elif isinstance(cdf, xr.DataArray): - helper_raise_value_error_cdf(cdf) - - def rps( observations, forecasts, @@ -568,27 +510,24 @@ def rps( The forecast of the event with dimension specified by ``member_dim``. Further requirements are specified based on ``category_edges``. category_edges : array_like, xr.Dataset, xr.DataArray, None - Category edges used to compute the CDFs. Similar to np.histogram, all but the - last (righthand-most) bin include the left edge and exclude the right edge. The - last bin includes both edges. - These ``category_edge`` must span the full range of the observations and - forecasts distribution. Forecasts, observations and category_edge are expected + Edges (left-edge inclusive) of the bins used to calculate the cumulative density function (cdf). Note that here the bins have to include the full range of observations and forecasts data. Effectively, negative infinity is appended to the left side of category_edges, and positive infinity is appended to the right side. Thus, N category edges produces N+1 bins. For example, specifying category_edges = [0,1] will compute the cdfs for bins [-inf, 0), [-inf, 1) and [-inf, inf). Note that the edges are right-edge exclusive. + Forecasts, observations and category_edge are expected in absolute units or probabilities consistently. ``category_edges`` decides how xs.rps interprets forecasts and observations. - - np.array (1d): will be internally converted and broadcasted to observations. + - np.array (1d): will be internally converted and broadcasted to observations. Use this if you wish to use the same category edges for all elements of both forecasts and observations. - xr.Dataset/xr.DataArray: edges of the categories provided as dimension ``category_edge`` with optional category labels as ``category_edge`` coordinate. Use xr.Dataset/xr.DataArray if edges - multi-dimensional and vary across dimensions. + multi-dimensional and vary across dimensions. Use this if your category edges vary across dimensions of forecasts and observations, but are the same for both. - tuple of np.array/xr.Dataset/xr.DataArray: same as above, where the first item is taken as ``category_edges`` for observations and the second item - for ``category_edges`` for forecasts. + for ``category_edges`` for forecasts. Use this if your category edges vary across dimensions of forecasts and observations, and are different for each. - None: expect than observations and forecasts are already CDFs containing - ``category_edge`` dimension. + ``category_edge`` dimension. Use this if your category edges vary across dimensions of forecasts and observations, and are different for each. dim : str or list of str, optional Dimension over which to mean after computing ``rps``. This represents a mean @@ -627,26 +566,23 @@ def rps( array([0.14814815, 0.7037037 , 1.51851852]) Coordinates: * y (y) int64 0 1 2 - forecasts_category_edge >> category_edges = xr.concat([ - ... xr.DataArray(0).expand_dims('category_edge').assign_coords(category_edge=[0]), - ... observations.quantile(q=[.33, .66]).rename({'quantile':'category_edge'}), - ... xr.DataArray(1).expand_dims('category_edge').assign_coords(category_edge=[1]) - ... ],'category_edge') + >>> category_edges = observations.quantile( + ... q=[.33, .66]).rename({'quantile': 'category_edge'}), >>> xs.rps(observations, forecasts, category_edges, dim='x') array([1.18518519, 0.85185185, 0.40740741]) Coordinates: * y (y) int64 0 1 2 - forecasts_category_edge Date: Tue, 9 Mar 2021 15:21:04 +0100 Subject: [PATCH 32/39] quick-start rps now equals brier --- docs/source/quick-start.ipynb | 560 ++++-------------------- xskillscore/core/probabilistic.py | 64 +-- xskillscore/tests/test_probabilistic.py | 5 +- 3 files changed, 131 insertions(+), 498 deletions(-) diff --git a/docs/source/quick-start.ipynb b/docs/source/quick-start.ipynb index 5a6f5284..d31375e7 100644 --- a/docs/source/quick-start.ipynb +++ b/docs/source/quick-start.ipynb @@ -527,20 +527,20 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 19, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\n", + "\n", "array(0.15555556)\n" ] } ], "source": [ - "rps = xs.rps(obs3>.5, fct3>.5, category_edges=np.array([0.0, 0.5, 1.0]))\n", + "rps = xs.rps(obs3>.5, fct3>.5, category_edges=np.array([0.5]))\n", "print(rps)" ] }, @@ -603,14 +603,6 @@ " * forecast_probability (forecast_probability) float64 0.1 0.3 0.5 0.7 0.9\n", " samples (forecast_probability) float64 1.0 5.0 0.0 11.0 3.0\n" ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/aaron.spring/Coding/xskillscore/xskillscore/core/probabilistic.py:843: RuntimeWarning: invalid value encountered in long_scalars\n", - " r[..., i] = N_o_f_in_bin / N_f_in_bin\n" - ] } ], "source": [ @@ -620,9 +612,32 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0.8229166666666666" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAARIAAAEGCAYAAACpcBquAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAkBUlEQVR4nO3deXgV5fn/8fedjSYsCRCQfVcUBESigloBrYItgnJ5uVTUtlLKFy12+aFSCkX9WqFYWyko5qetWncLRcQFK4r6U1IIhk0Wq6iQIA2IIIQIJLl/f8ygh5hlknPmzFnu13WdK3PmzDnzSSB3nueZmWdEVTHGmHCkBB3AGBP/rJAYY8JmhcQYEzYrJMaYsFkhMcaELS3oAA2Vm5ur3bp1CzqGMUlnzZo1e1S1TU2vxV0h6datG4WFhUHHMCbpiMintb1mXRtjTNiskBhjwmaFxBgTNiskxpiwWSExxoTNt6M2IvJXYBRQqqqn1vC6APcB3wcOAT9S1ff8ymNMPFlcVMKcZVvZua+cDjmZTBnRm0sHdozZffjZInkEGFnH6xcDJ7qPCcADPmYxJm4sLiph6qINlOwrR4GSfeVMXbSBxUUlMbsP3wqJqr4F7K1jkzHAY+ooAHJEpL1feYyJF3OWbaX8aOVx68qPVjJn2daI76Oy7AuOTSUSzj6CHCPpCOwIeV7srvsWEZkgIoUiUrh79+6ohDMmKDv3lTdofWP3oarsXnQXX65aFPY+giwkUsO6GmdZUtV8Vc1T1bw2bWo8Q9eYhNEhJ7NB6xu7DxEhd/QtZJ04OOx9BFlIioHOIc87ATsDymJMzJgyojeZ6anHrctMT2XKiN4R+fxVq1bRdfvLfCcthbTstqS36hj2PoIsJEuA68QxGNivqp8FmMeYmHDpwI7cPbYfHXMyEaBjTiZ3j+0XsaM2zzzzDP9etojpF3WN2D7ErzlbReQpYBiQC/wX+B2QDqCqC9zDv/NwjuwcAn6sqvVejZeXl6d20Z4xDaeqiAhVVVXs3buX3NzcBr1fRNaoal5Nr/l2HomqXl3P6wrc6Nf+jTHfWL58OdOmTWPp0qXk5uY2uIjUx85sNSZJqCpVVVW+fLYVEmMS2LHTJS644AIKCgpo27atL/uxQmJMgnrttdfo1q0bb7zxBgDOsKQ/rJAYk6DOOOMMrr/+ek4//XTf92WFxJgEs2LFCioqKsjOzub+++8nOzvb931aITEmgWzcuJHzzz+fP/3pT1Hdb9xN/myMqd2pp57Kk08+yZgxY6K6X2uRGJMA8vPz+eCDDwC46qqryMyM3HU5XlghSXKLi0o4Z9brdL/tRc6Z9XpE57ww0bF3716mT5/OfffdF1gG69oksWOT2xyb++LY5DZAxGfjMv5p1aoVK1eupHPnzvVv7BNrkSSxaEygY/yhqsyYMYN58+YB0KNHD9LT0wPLYy2SJBaNCXSMP6qqqtiwYQO5ublfX4wXJCskSaxDTiYlNRSNSE6gYyJLVfnqq6/IzMzkmWeeIS0tLfAiAta1SWp+T6BjIu8Xv/gFI0eO5PDhw2RkZJCSEhu/wtYiSWLHBlT9vu2BiZwhQ4aQlZVFRkZG0FGO49vERn6xiY1MsqmsrGTr1q306dMn0Bx1TWwUG+0iY0ytpk2bxllnnUVJSeye42NdG2Ni3OTJk+nRowcdO8Zul9NaJMbEoMOHD/Pwww+jqnTo0IEJEyYEHalOVkiMiUGPPfYY48ePp6CgIOgonljXxpgYNH78ePr06cOQIUOCjuKJtUiMiREHDhzghhtuYNeuXYgI55xzTtCRPLNCYkyM+OCDD1i4cCGrV68OOkqDWdfGmIBVVlaSmprKoEGD+Pjjj2nZsmXQkRrMWiTGBOjzzz9nyJAh/OMf/wCIyyICVkiMCVSTJk3Izs6mefPmQUcJi3VtjAlAaWkpLVu2pFmzZrz66qsxcQVvOKxFYkyUHTp0iHPPPffrk8zivYiAtUiMibqsrCwmT57MoEGDgo4SMVZIjImSjz76iMOHD9OnTx9uuummoONElBUSY6JAVbn66qspLy9n3bp1MTMhUaT4WkhEZCRwH5AKPKSqs6q9ng08DnRxs9yjqn/zM5MxQRARHnvsMY4ePZpwRQR8HGwVkVRgPnAx0Ae4WkSqz8xyI7BJVQcAw4A/ikhsTf1kTBjWr1//9UzvJ598Mv369Qs4kT/8LI1nAh+q6jZVPQI8DVS/j6ACzcUZtm4G7AUqfMxkTFQ98MADzJ49m/379wcdxVd+FpKOwI6Q58XuulDzgFOAncAG4GZVrar+QSIyQUQKRaRw9+7dfuU1JuLmzp1LQUEB2dnZQUfxlZ+FpKaD49UniB0BrAU6AKcB80SkxbfepJqvqnmqmtemTZtI5zQmot555x1GjBjBl19+SXp6ekzPbBYpfhaSYiD0HoKdcFoeoX4MLFLHh8DHwMk+ZjLGd3v27KGkpISDBw8GHSVq/Cwkq4ETRaS7O4B6FbCk2jbbgQsAROQEoDewzcdMxvjmyy+/BGDMmDGsXbuWDh06BJwoenwrJKpaAdwELAM2A8+q6vsiMlFEJrqb3QmcLSIbgOXAraq6x69MxvjljTfeoHv37qxcuRKAtLTkOkXL1+9WVV8CXqq2bkHI8k7gIj8zGBMNffv2ZeTIkZx00klBRwlE4p0ZY0wUFRYWoqq0bduWJ554gtatWwcdKRBWSIxppHXr1jF48GDmzp0bdJTAJVdHzpgI6t+/P/PmzWPcuHFBRwmctUiMaaAnn3yS4uJiRISJEyfSrFmzoCMFzgqJMQ2wZ88eJk2axO9///ugo8QU69oY0wC5ubm8/fbbSXt0pjbWIjHGgz/84Q88/vjjAPTr148mTZoEnCi2WCExph4VFRW88sorLFu2LOgoMcu6NsbUQlWpqKggPT2dpUuXWiukDtYiMaYWt912G2PHjuXo0aNkZWWRmpoadKSYZS0SY2rRvXt3Dh06ZAXEAyskxoSoqqpix44ddO3alYkTJ9b/BgN46NqIY5yIzHCfdxGRM/2PZkz0TZs2jUGDBrFr166go8QVLy2S+4Eq4HzgDuAAsBA4w8dcxgTihhtuIDc3lxNOOCHoKHHFy2DrWap6I/AVgKp+AdhM7yZhHD16lIULFwLQq1cvfv3rXyfEbTSjyUshOereWkIBRKQNTgvFmISQn5/P5ZdfzqpVq4KOEre8dG3mAv8E2orIXcDlwHRfUxkTRRMnTqRnz56ceaYN/TVWvS0SVX0CuAW4G/gMuFRVn/U7mDF+OnToEL/85S/Zt28fqampjBw5MuhIcc3LUZu/q+oWVZ2vqvNUdbOI/D0a4YzxS1FREQsWLODNN98MOkpC8NK16Rv6xB0vGeRPHGP8paqICOeccw7btm2jffv2QUdKCLW2SERkqogcAPqLyJcicsB9Xgo8H7WExkTI/v37Of/883n11VcBrIhEUK2FRFXvVtXmwBxVbaGqzd1Ha1WdGsWMxkREZWUlZWVllJWVBR0l4dTbtVHVqSLSEjgR+E7I+rf8DGZMpOzbt4/mzZvTqlUrCgoKSEmxa1Ujzctg63jgLZwbXd3ufp3pbyxjIqOsrIzvfve73HzzzQBWRHzi5ad6M87p8J+q6nBgILDb11TGREjTpk258sorGTt2bNBREpqXozZfqepXIoKINFHVLSLS2/dkxoRhx44dVFRU0L17d377298GHSfheSkkxSKSAywG/iUiXwA7/QxlTDhUlcsuu4yKigree+89685EgZfB1svcxZki8gaQDbziaypjwiAiPPjgg6iqFZEoqbOQiEgKsF5VTwVQVTsN0MSsLVu2sHr1aq699loGDbJzJqOpznKtqlXAOhHpEqU8xjTa3XffzS233MKBAweCjpJ0vIyRtAfeF5FVwNdn8qjq6PreKCIjgfuAVOAhVZ1VwzbDgD8D6cAeVR3qJbgx1S1YsICdO3fSvHnzoKMkHS+F5PbGfLB7Tc584EKgGFgtIktUdVPINjk4M7CNVNXtItK2MfsyyauwsJA5c+bw6KOPkpmZSc+ePYOOlJS8DLY2dlzkTOBDVd0GICJPA2OATSHb/BBYpKrb3X2VNnJfJklt2bKFwsJC9uzZQ6dOnYKOk7T8HNLuCOwIeV7srgt1EtBSRFaIyBoRua6mDxKRCSJSKCKFu3fbuXAGDh8+DMC4cePYuHGjFZGA+VlIapr0Uqs9T8OZkuAHwAhguoh86+7MqpqvqnmqmtemTZvIJzVx5e2336Znz56sXbsWgMzMzGADGW+FREQyG3E2azHQOeR5J759Ilsx8IqqlqnqHpxregY0cD8myXTt2pUBAwbQrl27oKMYl5eL9i4B1uKehCYip4nIEg+fvRo4UUS6i0gGcBVQ/X3PA98VkTQRyQLOAjY3IL9JIlu2bEFV6dKlCy+++KIVkhjipUUyE2fgdB+Aqq4FutX3JlWtAG7CuVp4M/Csqr4vIhNFZKK7zWacArUeWIVziHhjQ78Jk/iKioro378/+fn5QUcxNfBy+LdCVfc35j4fqvoS8FK1dQuqPZ8DzGnwh5ukMmDAAO644w6uvPLKoKOYGnhpkWwUkR8CqSJyooj8BXjX51zGAPD888+zZ88eUlJSuO2228jJyQk6kqmBl0Lyc5wJoA8DTwL7gV/4mMkYAEpLS7nmmmuYOXNm0FFMPbx0bXqr6jRgmt9hjAnVtm1bXnvtNfr37x90FFMPLy2Se0Vki4jcKSJ969/cmPDMnz+fpUuXAjB48GCysrICTmTq4+VOe8OBYTjTK+aLyAYRsSmnjC+OHj3Ko48+yuOPPx50FNMAolr9ZNM6Nhbph3P7zitVNcO3VHXIy8vTwsLCIHZtfFZVVUVKSgr79u2jadOmpKenBx3JhBCRNaqaV9NrXk5IO0VEZorIRmAezhEbu7DBRNTtt9/OddddR2VlJTk5OVZE4oyXwda/AU8BF6mqzdVqfJGenk5GRiCNXBMBDeraxALr2iQOVaW0tJQTTjjh6+eNOfHRREejujYi8qz7dYOIrA95bBCR9X6FNcljxowZDBo0iNJSZxoaKyLxq66uzc3u11HRCGKSzxVXXEFKSgo2NUT8q+sm4p+5i5NU9dPQBzApOvFMoqmsrOS1114DoF+/ftx+++3WEkkAXk5Iu7CGdRdHOohJDvPnz+fCCy9kzZo1QUcxEVRr10ZE/gen5dGj2phIc+Adv4OZxPSzn/2Mdu3a2X1nEkxdLZIngUtwJiO6JOQxSFXHRSGbSRCHDx/md7/7HWVlZTRp0oQrrrgi6EgmwuoqJKqqnwA3AgdCHohIK/+jmUTx7rvvctddd/Hqq68GHcX4pK6jNk/iHLFZgzNpc+iImAI9fMxlEsjw4cPZsmULvXr1CjqK8UldR21GuV+7q2oP9+uxhxURU6eDBw8yZswYVq5cCWBFJMF5udbmHBFp6i6PE5F77V7Apj4HDhzggw8+4NNPPw06iokCL9faPAAMEJEBOFf+Pgz8HbB79JpvKSsrIysri/bt27Nu3Tq7fiZJeDmPpEKdC3LGAPep6n04h4CNOc7Bgwc577zzmDbNmUzPikjy8NIiOSAiU4Frce5BkwrYNd7mW5o2bcrQoUM599xzg45iosxLIbkS52bfP1HVXe74iN0+wnxt165dALRr145777034DQmCF6mWtwFPAFki8go4CtVfcz3ZCYuVFVVMWrUKMaMGUO8TUlhIqfeFomIXIHTAlmBcy7JX0Rkiqr+w+dsJg6kpKRwzz33kJaWZhffJTEvXZtpwBmqWgogIm2A1wArJEls27ZtbNy4kdGjRzNs2LCg45iAeTlqk3KsiLg+9/g+k8B+85vfMGHCBMrKyoKOYmKAlxbJKyKyDGfeVnAGX1+qY3uTBPLz8ykuLqZp06ZBRzExwMtg6xTgQaA/MADIV9Vb/Q5mYs/GjRuZOHEiFRUVtGjRgj59+gQdycQIr12Ud4E3gdeBlf7FMbHsrbfe4oUXXmDnTruZgDmel2ttxgOrgMuAy4ECEfmJ38FM7KisrARg0qRJvP/++3TpYpdameN5aZFMAQaq6o9U9XpgEOCpayMiI0Vkq4h8KCK31bHdGSJSKSKXe4ttoqWgoIC+ffuydetWAHJycoINZGKSl0JSjDuhkesAsKO+N7mn0s/Hmd+1D3C1iHyrU+1uNxtY5iWwia7s7Gxyc3PtRt6mTl6O2pQA/xaR53EmNBoDrBKRXwGoam3nRJ8JfKiq2wBE5Gn3vZuqbfdzYCFwRsPjG7/s2LGDzp07c8opp/D222/byWamTl5aJB8Bi3GKCMDzwGc4VwDXdRVwR45vuRS7674mIh1xxl4W1BVARCaISKGIFO7evdtDZBOOoqIievfuzWOPOVdCWBEx9am3RaKqtzfys2v631f9Yow/A7eqamVd/1lVNR/IB+eWnY3MYzw69dRTmTx5MhdfbHcdMd546do0VjHQOeR5J6D6ccM84Gm3iOQC3xeRClVd7GMuU4vly5dzxhln0KJFC2bNmhV0HBNH/DzVfTVwooh0F5EM4CqcW1t8zZ3/tZuqdsO5dmeSFZFg7Nq1i1GjRvHb3/426CgmDvnWIlHVChG5CedoTCrwV1V9X0Qmuq/XOS5ioqtdu3YsWbKEwYMHBx3FxCGpbw4JETkJZ97WE1T1VBHpD4xW1f+NRsDq8vLytLCwMIhdJ6RHHnmE7t27M3SoTcFr6iYia1Q1r6bXvHRt/i8wFTgKoKrrcbopJs4dOXKEe+65h7/85S9BRzFxzkvXJktVV1U7qlLhUx4TJapKRkYGr7/+Oi1atAg6jolzXloke0SkJ+6hW/c09s98TWV8dc899zB58mRUlbZt2/Kd73wn6EgmznlpkdyIcw7HySJSAnwM2E3E49iuXbsoLS2lsrKStDQ/zwAwyaLewdavN3Tutpeiqgfq3dhHNtjaOKrKl19+SXZ2NqpKVVUVqampQccycaSuwVYvkz/PqPYcAFW9IyLpTFTceeedPP7446xcuZLWrVtbETER5aVdGzop53eAUcBmf+IYv1x44YXs37+fli1bBh3FJCDPXZuv3yDSBFiiqiP8iVQ369p4V1VVxerVqznrrLOCjmISQLjnkVSXBfQIL5KJhrlz53L22Wezdu3aoKOYBOdljGQD31y1mwq0AWx8JA789Kc/pVmzZgwYMCDoKCbBeRkjGRWyXAH8V1XthLQYVVFRwdy5c7npppto2rQp48ePDzqSSQJ1dm1EJAV4UVU/dR8lVkRi2/Lly/n1r3/Niy++GHQUk0TqbJGoapWIrBORLqq6PVqhTOONGDGCoqIiTjvttKCjmCTiZbC1PfC+iCwXkSXHHn4HM96Vl5czbtw4NmzYAGBFxESdlzGSxk61aKKktLSUt956i4suuoh+/foFHcckIS+F5PvVb9EpIrNx7rxnAnTkyBEyMjLo2rUrmzdvtvvwmsB46dpcWMM6mxU4YAcPHmT48OHMnj0bwIqICVStLRIR+R9gEtBDRNaHvNQceMfvYKZumZmZnHTSSfTq1SvoKMbU2bV5EngZuBsIvd3mAVXd62sqU6vPP/+clJQUWrZsyd/+9reg4xgD1FFIVHU/sB+4OnpxTF2qqqoYOXIkWVlZrFixwm5cZWJGws1qs7iohDnLtrJzXzkdcjKZMqI3lw7sWP8b40BKSgrTp08nKyvLioiJKQlVSBYXlTB10QbKj1YCULKvnKmLnHMr4rmYlJSU8J///Idhw4YxevTooOMY8y1+3iAr6uYs2/p1ETmm/Gglc5ZtDShRZEyePJmrrrqKQ4cOBR3FmBolVItk577yBq2PFwsWLGD79u1kZWUFHcWYGiVUi6RDTmaD1sey//znP0ydOpWqqiratGnDoEGDgo5kTK0SqpBMGdGbzPTj5yLNTE9lyojeASVqvH/+85889NBDFBcXBx3FmHo1eKrFoNU31WK8H7VRVUQEVeW///0v7dq1CzqSMUCYs8jHm0sHdoyrwhGqqKiICRMmsHDhQrp06WJFxMSNhOraxLuqqiqOHDnC0aNHg45iTIMkXIskHu3Zs4fc3FwGDRpEUVERKSlW30188fV/rIiMFJGtIvKhiNxWw+vXiMh69/GuiCTdLMVFRUX07NmT5557DsCKiIlLvv2vFZFUYD7OlAN9gKtFpE+1zT4Ghqpqf+BOnHsMJ5WTTz6ZH/7wh5x77rlBRzGm0fz883cm8KGqblPVI8DTwJjQDVT1XVX9wn1aAHTyMU9MWblyJV999RWZmZk88MADtG/fPuhIxjSan4WkI7Aj5Hmxu642N+BMW/AtIjJBRApFpHD37t0RjBiMnTt3cv755zNt2rSgoxgTEX4OttZ0eWqNJ62IyHCcQlJj+15V83G7PXl5efF14ksNOnTowBNPPMHw4cODjmJMRPjZIikGOoc87wTsrL6RiPQHHgLGqOrnPuYJ3LPPPsuaNWsAGDt2rN3Q2yQMPwvJauBEEekuIhnAVcBxt7EQkS7AIuBaVf3AxyyBO3z4MFOnTuWuu+4KOooxEedb10ZVK0TkJmAZzj2D/6qq74vIRPf1BcAMoDVwvztRT0Vtp+DGuyZNmvDGG2/QunXroKMYE3EJd61NrLn//vspLS1l5syZQUcxJixJda1NLFFV3nvvPUpLS6moqCAtzX7cJjHZ/2yflJeXk5mZyYMPPkhlZaUVEZPQ7HxsH8yaNYuzzz6b/fv3k5qaSkZGRtCRjPGV/Zn0wemnn862bdto1qxZ0FGMiQprkUSIqrJp0yYALrroIvLz80lNTa3nXcYkBiskEXLfffcxcOBANm7cGHQUY6LOujYRcv3111NVVUXfvn2DjmJM1FmLJAyVlZU8/PDDVFZW0rJlS371q1/ZHfBMUrJCEoaXX36Z8ePHs2TJkvo3NiaBWdcmDKNGjeLNN9/kvPPOCzqKMYGyFkkDHTlyhEmTJvHRRx8BWBExBiskDbZ9+3aee+45VqxYEXQUY2KGdW08qqqqIiUlhV69erF161ZatWoVdCRjYoa1SDw4dOgQI0aM4MEHHwSwImJMNVZIPEhLSyMrK4usrKygoxgTk6xrU4f9+/eTlpZG06ZNWbx4sZ0jYkwtrEVSi8rKSkaOHMnll1/+9Y29jTE1sxZJLVJTU7nxxhvJzs62ImJMPayQVFNaWsr27dvJy8tj3LhxQccxJi5Y16aa8ePHM3r0aMrLy4OOYkzcsBZJNfPnz+fTTz8lMzMz6CjGxA1rkQCffPIJs2fPRlXp3Lmz3dDbmAayQgI88sgjzJo1i5KSkqCjGBOXrJAAM2bMoKioiE6dOgUdxZi4lLSFZNOmTXzve99j9+7dpKSk0K1bt6AjGRO3kraQ7N69m08++YS9e/cGHcWYuJd0R20OHjxIs2bNGDp0KJs3byY9PT3oSMbEvaRqkaxbt44ePXrw0ksvAVgRMSZCkqqQdO/enQsuuMBmejcmwpKikKxbt46KigpatGjBU089RdeuXYOOZExCSfhCsmPHDoYMGcL06dODjmJMwvJ1sFVERgL3AanAQ6o6q9rr4r7+feAQ8CNVfS+cfS4uKmHOsq3s3FdOh5xMpozozbx587jkkkvC+VhjTB18a5GISCowH7gY6ANcLSJ9qm12MXCi+5gAPBDOPhcXlTB10QZK9pVT9uEqPvnoA6Yu2kCrgSNo06ZNOB9tjKmDn12bM4EPVXWbqh4BngbGVNtmDPCYOgqAHBFp39gdzlm2lfKjlVQdPczeZfP54s1HKT9ayZxlWxv/XRhj6uVn16YjsCPkeTFwlodtOgKfhW4kIhNwWix06dKl1h3u3Odc+p+S3oQTrrqL1GatjltvjPGHny2SmqYV00Zsg6rmq2qequbV1UXpkPPNpf/prTuR0iTrW+uNMZHnZyEpBjqHPO8E7GzENp5NGdGbzPTU49ZlpqcyZUTvxn6kMcYDPwvJauBEEekuIhnAVUD1u20vAa4Tx2Bgv6p+Vv2DvLp0YEfuHtuPjjmZCNAxJ5O7x/bj0oEdG/1NGGPq59sYiapWiMhNwDKcw79/VdX3RWSi+/oC4CWcQ78f4hz+/XG4+710YEcrHMZEma/nkajqSzjFInTdgpBlBW70M4Mxxn8Jf2arMcZ/VkiMMWGzQmKMCZsVEmNM2MQZ74wfIrIb+NTDprnAHp/jhMsyhi/W80HsZ/Sar6uq1nhGaNwVEq9EpFBV84LOURfLGL5YzwexnzES+axrY4wJmxUSY0zYErmQ5AcdwAPLGL5YzwexnzHsfAk7RmKMiZ5EbpEYY6LECokxJmxxX0hEZKSIbBWRD0XkthpeFxGZ676+XkROj8GM17jZ1ovIuyIyIJbyhWx3hohUisjl0czn7rvejCIyTETWisj7IvJmLOUTkWwReUFE1rn5wr7SvYH5/ioipSKysZbXw/s9UdW4feBMT/AR0APIANYBfapt833gZZzZ2AYD/47BjGcDLd3li6OZ0Uu+kO1ex7ma+/IY/BnmAJuALu7ztjGW7zfAbHe5DbAXyIhixvOA04GNtbwe1u9JvLdIoj7BtB8ZVfVdVf3CfVqAM1NczORz/RxYCJRGMdsxXjL+EFikqtsBVDWaOb3kU6C5ewuWZjiFpCJaAVX1LXeftQnr9yTeC0ltk0c3dBs/NXT/N+D8ZYiWevOJSEfgMmABwfDyMzwJaCkiK0RkjYhcF7V03vLNA07BmUp0A3CzqlZFJ54nYf2e+DqxURREbIJpH3nev4gMxykk5/qaqNpua1hXPd+fgVtVtdL5gxp1XjKmAYOAC4BMYKWIFKjqB36Hw1u+EcBa4HygJ/AvEXlbVb/0OZtXYf2exHshifoE043gaf8i0h94CLhYVT+PUjbwli8PeNotIrnA90WkQlUXRyWh93/nPapaBpSJyFvAACAahcRLvh8Ds9QZkPhQRD4GTgZWRSGfF+H9nkRrsMenAaQ0YBvQnW8GufpW2+YHHD+ItCoGM3bBmbf27Fj8GVbb/hGiP9jq5Wd4CrDc3TYL2AicGkP5HgBmussnACVAbpR/jt2ofbA1rN+TuG6RaEATTPuQcQbQGrjf/atfoVG6WtRjvkB5yaiqm0XkFWA9UIVzr+kaD3UGkQ+4E3hERDbg/LLeqqpRm1pARJ4ChgG5IlIM/A5ID8kX1u+JnSJvjAlbvB+1McbEACskxpiwWSExxoTNCokxJmxWSIwxYbNCEudEZLKIbBaRJ+rYZpiILI1mrtqIyOhjV8eKyKUi0ifktTtE5HtRzDJMRM6O1v4SWVyfR2IAmIRzNuzHQQfxQlWXAEvcp5cCS3Gu2kVVZ0R6fyKSpqq1XRw3DDgIvBvp/SYba5HEMRFZgHPp+hIR+aWInOnOZ1Lkfu1dw3uGunN2rHW3a+6unyIiq925KG6vZX8HReSPIvKeiCwXkTbu+tNEpMB97z9FpKW7frKIbHLXP+2u+5GIzHNbAqOBOW6WniLyiIhcLiIXi8izIfsdJiIvuMsXichKN8NzItKshpwrROT37pwkN4vIJSLyb/f7fU1EThCRbsBE4Jfu/r8rIm1EZKH7c1gtIueE8++TVKJ5iq49fDnt+RPcU62BFkCau/w9YKG7PAxY6i6/AJzjLjfDaZVehDMBsOD8cVkKnFfDvhS4xl2eAcxzl9cDQ93lO4A/u8s7gSbuco779Uch73uEkNPtjz13M20HmrrrHwDG4Vzn81bI+luBGTXkXAHcH/K8Jd+cfDke+KO7PBP4PyHbPQmc6y53ATYH/e8bLw/r2iSWbOBRETkR55c+vYZt3gHudcdUFqlqsYhchFNMitxtmgEn4vzShqoCnnGXHwcWiUg2TpE4NiPZo8Bz7vJ64AkRWQws9vpNqHPK+SvAJSLyD5zrQG4BhgJ9gHfcSwkygJW1fMwzIcudgGfc+TUygNq6gd8D+oRc4dxCRJqr6gGv2ZOVFZLEcifwhqpe5jbdV1TfQFVniciLONdVFLiDmwLcraoPNnB/9V1f8QOcmblGA9NFpG8DPvsZ4EacyXhWq+oBd1Kgf6nq1R7eXxay/BfgXlVdIiLDcFoiNUkBhqhqeQNyGmyMJNFk41xVCk4X4ltEpKeqblDV2UAhzqXsy4CfHBtvEJGOItK2hren4HQ9wJmR7P+p6n7gCxH5rrv+WuBNEUkBOqvqGzitiRyclk6oA0DzWr6XFThTA/6Ub1oXBcA5ItLLzZklIifV8v5QoT+X6+vY/6vATceeiMhpHj7bYIUk0fwBuFtE3sG5CrUmvxCRjSKyDigHXlbVV3HGB1a6V6f+g5p/wcuAviKyBmeCnjvc9dfjDJquB05z16cCj7ufVwT8SVX3Vfu8p4Ep7iBoz9AXVLUSZ6zmYvcrqrobp0A+5e6rAKcQ1mcm8JyIvM3xN8t+Abjs2GArMBnIcweHN+EMxhoP7Opf45mIHFTVbx0lMcZaJMaYsFmLxBgTNmuRGGPCZoXEGBM2KyTGmLBZITHGhM0KiTEmbP8fEYuHxSgxtMgAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], "source": [ "# ROC for probabilistic forecasts and bin_edges='continuous' default\n", "roc = xs.roc(obs3 > 0.5, (fct3 > 0.5).mean(\"member\"), return_results='all_as_metric_dim')\n", @@ -644,7 +659,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ @@ -658,7 +673,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 25, "metadata": {}, "outputs": [ { @@ -979,7 +994,8 @@ " grid-template-columns: 125px auto;\n", "}\n", "\n", - ".xr-attrs dt, dd {\n", + ".xr-attrs dt,\n", + ".xr-attrs dd {\n", " padding: 0;\n", " margin: 0;\n", " float: left;\n", @@ -1028,16 +1044,16 @@ " observations_category_bounds (observations_category) <U10 '[0.0, 0.5)' '...\n", " forecasts_category_bounds (forecasts_category) <U10 '[0.0, 0.5)' '[0....\n", " * observations_category (observations_category) int64 1 2\n", - " * forecasts_category (forecasts_category) int64 1 2
  • " ], "text/plain": [ "\n", @@ -1057,7 +1073,7 @@ " * forecasts_category (forecasts_category) int64 1 2" ] }, - "execution_count": 24, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } @@ -1068,7 +1084,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 26, "metadata": {}, "outputs": [ { @@ -1109,7 +1125,7 @@ " \n", " observations_category_bounds\n", " [0.0, 0.5)\n", - " [0.5, 1.0)\n", + " [0.5, 1.0]\n", " \n", " \n", " forecasts_category\n", @@ -1127,7 +1143,7 @@ " \n", " \n", " 2\n", - " [0.5, 1.0)\n", + " [0.5, 1.0]\n", " 5.33\n", " 4.67\n", " \n", @@ -1141,17 +1157,17 @@ "observations_category_bounds [0.0, 0.5) \n", "forecasts_category forecasts_category_bounds \n", "1 [0.0, 0.5) 5.33 \n", - "2 [0.5, 1.0) 5.33 \n", + "2 [0.5, 1.0] 5.33 \n", "\n", " \n", "observations_category 2 \n", - "observations_category_bounds [0.5, 1.0) \n", + "observations_category_bounds [0.5, 1.0] \n", "forecasts_category forecasts_category_bounds \n", "1 [0.0, 0.5) 4.67 \n", - "2 [0.5, 1.0) 4.67 " + "2 [0.5, 1.0] 4.67 " ] }, - "execution_count": 25, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } @@ -1187,7 +1203,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -1207,7 +1223,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 28, "metadata": {}, "outputs": [ { @@ -1227,7 +1243,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 29, "metadata": {}, "outputs": [ { @@ -1247,7 +1263,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 30, "metadata": {}, "outputs": [ { @@ -1274,7 +1290,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -1287,7 +1303,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 32, "metadata": {}, "outputs": [ { @@ -1307,7 +1323,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 33, "metadata": {}, "outputs": [ { @@ -1327,7 +1343,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 34, "metadata": {}, "outputs": [ { @@ -1347,7 +1363,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 35, "metadata": {}, "outputs": [ { @@ -1367,9 +1383,32 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 36, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0.5035528250988777" + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAARIAAAEGCAYAAACpcBquAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAlKUlEQVR4nO3deXwV5dnw8d+VjSYsCRiQfRVRFBCJCqKCWgVbBOXlU9Gito+U8qjFtr6olMKD+lig2EUKinm1VetupYi4YEVRq6QQDZssFlEhQRoQwxohy/X+MQMewkkyOSdzJifn+n4++WRmzpy5L0LOlXvuuRdRVYwxJhpJQQdgjIl/lkiMMVGzRGKMiZolEmNM1CyRGGOilhJ0AHWVnZ2tXbt2DToMYxLOhx9+uFtVW4d7Le4SSdeuXcnPzw86DGMSjoh8Ud1rdmtjjImaJRJjTNQskRhjomaJxBgTNUskxpio+ZZIROTPIlIsIuureV1EZK6IbBGRtSJytl+xGGNOtKigiMGz3qLbXa8weNZbLCooivhaftZIHgOG1/D6FUBP92sC8JCPsRhjQiwqKGLKwnUUlZSiQFFJKVMWros4mfiWSFT1XWBPDaeMAp5QRx6QJSLt/IrHGPOtOUs3U1pWQcXBrzk6lUhpWQVzlm6O6HpBtpF0ALaH7Be6x04gIhNEJF9E8nft2hWT4IxpzHaUlKKq7Fp4H/tWLjzueCSCTCQS5ljYWZZUNVdVc1Q1p3XrsD10jTF10D4rHREhe+QdZPQceNzxSASZSAqBTiH7HYEdAcViTMJYuXIlXba9xndSkkjJbENqK+dGID01mcnDekV0zSATyWLgBvfpzUBgr6p+GWA8xiSE5557jn8tXci0y7vQISsdATpkpTNzdB+u6h+2daFW4tecrSLyDDAUyAb+A/wPkAqgqgtERIB5OE92DgE/VtVaR+Pl5OSoDdozpu5UFRGhsrKSPXv2kJ2dXaf3i8iHqpoT7jXfRv+q6rW1vK7ALX6Vb4z51rJly5g6dSpLliwhOzu7zkmkNtaz1ZgEoapUVlb6cm1LJMY0Yke7S1x66aXk5eXRpk0bX8qxRGJMI/Xmm2/StWtX3n77bQCcZkl/WCIxppE655xzuPHGGzn7bP+HsVkiMaaRWb58OeXl5WRmZvLggw+SmZnpe5mWSIxpRNavX88ll1zCH/7wh5iWG3eTPxtjqnfmmWfy9NNPM2rUqJiWazUSYxqB3NxcPvnkEwDGjh1LenpkY2YiZYnEmDi3Z88epk2bxgMPPBBYDHZrY0yca9WqFStWrKBTp061n+wTq5EYE4dUlenTpzNv3jwAunfvTmpqamDxWI3EmDhUWVnJunXryM7OPjYYL0iWSIyJI6rKN998Q3p6Os899xwpKSmBJxGwWxtj4srPf/5zhg8fzuHDh0lLSyMpqWF8hK1GYkwcGTRoEBkZGaSlpQUdynF8m9jILzaxkUk0FRUVbN68md69ewcaR00TGzWMepExplpTp07lvPPOo6go8gWs/Ga3NsY0cJMmTaJ79+506BDZfKqxYDUSYxqgw4cP8+ijj6KqtG/fngkTJtR7GfGyZKcxJkJPPPEE48ePJy8vz5frx82SncaYyI0fP55//vOfDBo0yJfrH12yM1S8LtlpjAmxf/9+brrpJnbu3ImIMHjwYN/Kqm5pznhcstMYE+KTTz7hxRdfZNWqVb6XVd3SnPG4ZKcxBqefCMCAAQP47LPPuPLKK30vc/KwXqSnJh93LF6X7DQm4X311VcMGjSIv/3tbwC0bNkyJuVe1b8DM0f3qbclO60fiTEBatKkCZmZmTRv3jzmZV/Vv0PEiaMqSyTGBKC4uJiWLVvSrFkz3njjjQYxgjcadmtjTIwdOnSICy644Fgns3hPImA1EmNiLiMjg0mTJjFgwICgQ6k3lkiMiZFPP/2Uw4cP07t3b2699dagw6lXlkiMiQFV5dprr6W0tJQ1a9Y0mAmJ6ouviUREhgMPAMnAI6o6q8rrmcCTQGc3lvtV9S9+xmRMEESEJ554grKyskaXRMDHxlYRSQbmA1cAvYFrRaTqzCy3ABtUtR8wFPidiDSsqZ+MicLatWuPzfR+2mmn0adPn4Aj8oefqfFcYIuqblXVI8CzQNV1BBVoLk6zdTNgD1DuY0zGxNRDDz3E7Nmz2bt3b9Ch+MrPRNIB2B6yX+geCzUPOB3YAawDblPVyqoXEpEJIpIvIvm7du3yK15j6t3cuXPJy8sjMzMz6FB85WciCfdwvOoEscOA1UB74Cxgnoi0OOFNqrmqmqOqOa1bt67vOI2pV++//z7Dhg1j3759pKamNuiZzeqLn4mkEAhdQ7AjTs0j1I+BherYAnwGnOZjTMb4bvfu3RQVFXHgwIGgQ4kZPxPJKqCniHRzG1DHAournLMNuBRARE4GegFbfYzJGN/s27cPgFGjRrF69Wrat28fcESx41siUdVy4FZgKbAReF5VPxaRiSIy0T3tXuB8EVkHLAPuVNXdfsVkjF/uzX2ek9p2pN319zN41lssWfefoEOKKV/7kajqq8CrVY4tCNneAVzuZwzG+G1RQRFPbKokrdvZpLTqcGz+U6DeRtc2dI2vZ4wxMZSfn89vX99EWVpzWl85meR051lBNPOfxiNLJMZEaM2aNQwcOJDNy54L+3qk85/GI0skxkSob9++zJs3j54XVu1n6Yh0/tN4ZInEmDp6+umnKSwsRESYOHEid408q17nP41HlkiMqYPdu3dz880385vf/ObYsfqe/zQe2TQCxtRBdnY27733Hqeeeupxx+tz/tN4ZDUSYzz47W9/y5NPPglAnz59aNKkScARNSyWSIypRXl5Oa+//jpLly4NOpQGy25tjKmGqlJeXk5qaipLliyxWkgNrEZiTDXuuusuRo8eTVlZGRkZGSQnJ9f+pgRlNRJjqtGtWzcOHTpkCcQDSyTGhKisrGT79u106dKFiRMn1v4GA3i4tRHHOBGZ7u53FpFz/Q/NmNibOnUqAwYMYOfOnUGHEle81EgeBCqBS4B7gP3Ai8A5PsZlTCBuuukmsrOzOfnkk4MOJa54aWw9T1VvAb4BUNWvAZvp3TQaZWVlvPjiiwCccsop3H777Y1iGc1Y8pJIytylJRRARFrj1FCMaRRyc3MZM2YMK1euDDqUuOXl1mYu8HegjYjcB4wBpvkalTExNHHiRHr06MG551rTX6RqrZGo6lPAHcBM4EvgKlV93u/AjPHToUOH+MUvfkFJSQnJyckMHz486JDimpenNn9V1U2qOl9V56nqRhH5ayyCM8YvBQUFLFiwgHfeeSfoUBoFL7c2Z4TuuO0lA/wJxxh/qSoiwuDBg9m6dSvt2rWr8fxFBUXMWbqZHSWltM9KZ/KwXgk9yrc61dZIRGSKiOwH+orIPhHZ7+4XAy/FLEJj6snevXu55JJLeOONNwA8JZEpC9dRVFKKwrFJnRcVFMUg2vhSbSJR1Zmq2hyYo6otVLW5+3WSqk6JYYzG1IuKigoOHjzIwYMHPZ0/Z+lmSssqjjuWaJM6e1XrrY2qThGRlkBP4Dshx9/1MzBj6ktJSQnNmzenVatW5OXlkZTkbaxqdZM3J9Kkzl55aWwdD7yLs9DV3e73Gf6GZUz9OHjwIBdeeCG33XYbgOckAtVP3pxIkzp75eWnehtOd/gvVPVioD+wy9eojKknTZs25ZprrmH06NF1fu/kYb0SflJnr7w8tflGVb8REUSkiapuEhH7SZoGbfv27ZSXl9OtWzd+/etfR3SNo09n7KlN7bwkkkIRyQIWAf8Qka+BHX4GZUw0VJWrr76a8vJyPvroozrdzlSV6JM6e+WlsfVqd3OGiLwNZAKv+xqVMVEQER5++GFUNaokYryrMZGISBKwVlXPBFBV6wZoGqxNmzaxatUqrr/+egYMsD6TsVRjulbVSmCNiHSOUTzGRGzmzJnccccd7N+/P+hQEo6XNpJ2wMcishI41pNHVUfW9kYRGQ48ACQDj6jqrDDnDAX+CKQCu1V1iJfAjalqwYIF7Nixg+bNmwcdSsLxkkjujuTC7pic+cBlQCGwSkQWq+qGkHOycGZgG66q20SkTSRlmcSVn5/PnDlzePzxx0lPT6dHjx5Bh5SQvDS2Rtouci6wRVW3AojIs8AoYEPIOdcBC1V1m1tWcYRlmQS1adMm8vPz2b17Nx07dgw6nITlZ5N2B2B7yH6heyzUqUBLEVkuIh+KyA3hLiQiE0QkX0Tyd+2yvnAGDh8+DMC4ceNYv369JZGA+ZlIwk16qVX2U3CmJPg+MAyYJiKnnvAm1VxVzVHVnNatW9d/pCauvPfee/To0YPVq1cDkJ5uXdaD5imRiEh6BL1ZC4FOIfsdObEjWyHwuqoeVNXdOGN6+tWxHJNgunTpQr9+/Wjbtm3QoRiXl0F7VwKrcTuhichZIrLYw7VXAT1FpJuIpAFjgarvewm4UERSRCQDOA/YWIf4TQLZtGkTqkrnzp155ZVXLJE0IF5qJDNwGk5LAFR1NdC1tjepajlwK85o4Y3A86r6sYhMFJGJ7jkbcRLUWmAlziPi9XX9R5jGr6CggL59+5Kbmxt0KCYML49/y1V1byTrfKjqq8CrVY4tqLI/B5hT54ubhNKvXz/uuecerrnmmqBDMWF4qZGsF5HrgGQR6SkifwI+8DkuYwB46aWX2L17N0lJSdx1111kZWUFHZIJw0si+RnOBNCHgaeBvcDPfYzJGACKi4v54Q9/yIwZM4IOxdTCy61NL1WdCkz1OxhjQrVp04Y333yTvn37Bh2KqYWXGsnvRWSTiNwrImfUfrox0Zk/fz5LliwBYODAgWRkZAQckamNl5X2LgaG4kyvmCsi60QksimnjKlFWVkZjz/+OE8++WTQoZg6ENWqnU1rOFmkD87yndeoappvUdUgJydH8/Pzgyja+KyyspKkpCRKSkpo2rQpqampQYdkQojIh6qaE+41Lx3STheRGSKyHpiH88TGBjaYenX33Xdzww03UFFRQVZWliWROOOlsfUvwDPA5apqc7UaX6SmppKWFkgl19SDOt3aNAR2a9N4qCrFxcWcfPLJx/Yj6fhoYiOiWxsRed79vk5E1oZ8rRORtX4FaxLH9OnTGTBgAMXFzjQ0lkTiV023Nre530fEIhCTeH7wgx+QlJSETQ0R/6pNJKr6pbt5s6reGfqaiMwG7jzxXcaEt6igiDlLN1O05wBN92zmvluu46r+fejTp4/vZdriVv7z0iHtsjDHrqjvQEzjtaigiCkL11FUUsq+j15hw6N38Iv5C1lUUBSTMhUoKillysJ1vpaZyGpqI/lvEVkH9KrSRvIZzrB/YzyZs3QzpWUVADQ/6wqyR96JZndnztLNMSnzqNKyCl/LTGQ1tZE8DbwGzATuCjm+X1X3+BqVaVSKdu+jZMXztDjv/5CU9h2ann4hADtKSn0rs7pr+1lmIqvp1kZV9XPgFmB/yBci0sr/0Exj0Wzvp+xd8RzffP7RccfbZ/k312p11/azzERWW41kBPAhzqTNoc/mFOjuY1ymEfnfm8dye1pLKpqffOxYemoyk4fVdRpg7yYP68WUheuOu73xu8xEVm2NRFVHuN+7qWp39/vRL0sipkYHDhxg1KhRrFixgqv6d+B344fRISsdATpkpTNzdB9fn6Bc1b8DM0f3iWmZiazWLvIiMhhYraoHRWQccDbwx6OLWhkTzv79+/nkk0/44osvGDRoEFf17xDzD3EQZSYqL2NtHgL6iUg/nJG/jwJ/BWyNXnOCgwcPkpGRQbt27VizZo2Nn0kQXvqRlKszIGcU8ICqPgDYKs3mBAcOHOCiiy5i6lRnMj1LIonDS41kv4hMAa7HWYMmGbAx3uYETZs2ZciQIVxwwQVBh2JizEsiuQZnse//UtWdItIZWz7ChNi5cycAbdu25fe//33A0Zgg1JpI3OTxFHCOiIwAVqrqE/6HZiIVyzEmlZWVjBgxguTkZPLy8mwEb4Ly8tTmBzg1kOU4fUn+JCKTVfVvPsdmInB0jMnR/hNHx5gAviSTpKQk7r//flJSUiyJJDAvtzZTgXNUtRhARFoDbwKWSBqgmsaY1Gci2bp1K+vXr2fkyJEMHTq03q5r4pOXpzZJR5OI6yuP7zMBiNUYk1/96ldMmDCBgwcP1ut1TXzyUiN5XUSW4szbCk7j66s1nG8C1D4rnaIwSaO+x5jk5uZSWFhI06ZN6/W6Jj55WddmMvAw0BfoB+RWnejINByTh/UiPTX5uGP1NcZk/fr1TJw4kfLyclq0aEHv3r2jvqZpHLzeonwAvAO8BazwLxwTLT/HmLz77ru8/PLL7NhhiwmY49U6i7yIjAem4yQRwekaf4+q/tn/8E5ks8jHXkVFBcnJTi2npKSErKysYAMygYhqgSxgMtBfVX+kqjcCA/A4X6uIDBeRzSKyRUTuquG8c0SkQkTGeLmuiZ28vDzOOOMMNm92ZhazJGLC8ZJICnEnNHLtB7bX9ia3K/18nPldewPXisgJN9XuebOBpV4CNrGVmZlJdna2LeRtauTlqU0R8C8ReQlnQqNRwEoR+SWAqlbXJ/pcYIuqbgUQkWfd926oct7PgBeBc+oevvHL9u3b6dSpE6effjrvvfeedTYzNfJSI/kUWISTRABeAr7EGQFc0yjgDhxfcyl0jx0jIh2Aq4EFNQUgIhNEJF9E8nft2uUhZBONgoICevXqxRNPOCMhLImY2ngZa3N3hNcO99tXtWX3j8CdqlpR0y+rquYCueA0tkYYj/HozDPPZNKkSVxxRfhVR2y9GFOVl1ubSBUCnUL2OwJVnxvmAM+6SSQb+J6IlKvqIh/jMtVYtmwZ55xzDi1atGDWrFlhz4n1WB4TH/zs6r4K6Cki3UQkDRgLLA49wZ3/tauqdsUZu3OzJZFg7Ny5kxEjRvDrX/+6xvNsvRgTjm81ElUtF5FbcZ7GJAN/VtWPRWSi+3qN7SImttq2bcvixYsZOHBgjefZejEmHC/TCJyKM2/ryap6poj0BUaq6v/W9l5VfZUq43KqSyCq+iNPEZt69dhjj9GtWzeGDBnCZZeFW531eLEay2Pii5dbm/8HTAHKAFR1Lc5tiolzR44c4f777+dPf/qT5/f4OZbHxC8vtzYZqrqyylOVcp/iMTGiqqSlpfHWW2/RokULz+872qBqT21MKC+JZLeI9MB9dOt2Y//S16iMr+6//36++OIL5s6dS5s2ber8flsvxlTlJZHcgtOH4zQRKQI+A8b5GpXx1c6dOykuLqaiooKUFD97AJhEUevo32MnijTFmS1tf60n+yheRv82tE5bqsq+ffvIzMxEVamsrDw2otcYL2oa/evlqc30KvsAqOo99RJdI9QQO23de++9PPnkk6xYsYKTTjrJkoipV17qtaGTcn4HGAFs9CecxiFWEzDXxWWXXcbevXtp2bJlIOWbxs3LWJvfhe6LyP1U6aFqjtdQOm1VVlayatUqzjvvPAYNGsSgQYNiWr5JHJF0kc8Autd3II1JdZ2zYt1pa+7cuZx//vmsXr06puWaxOOljWQd347aTQZaA9Y+UoPJw3od10YCwXTa+slPfkKzZs3o169fTMs1icdLG8mIkO1y4D+qah3SahBkp63y8nLmzp3LrbfeStOmTRk/frzvZRpTYyIRkSTgFVU9M0bxNBpBddpatmwZt99+O926dePqq6+OefkmMdWYSFS1UkTWiEhnVd0Wq6BM5IYNG0ZBQQFnnXVW0KGYBOKlsbUd8LGILBORxUe//A7MeFdaWsq4ceNYt87pq2JJxMSalzaSSKdaNDFSXFzMu+++y+WXX06fPn2CDsckIC+J5HtVl+gUkdk4K++ZAB05coS0tDS6dOnCxo0bbR1eExgvtzbhZrsJPyuwiZkDBw5w8cUXM3v2bABLIiZQ1dZIROS/gZuB7iKyNuSl5sD7fgdmapaens6pp57KKaecEnQoxtR4a/M08BowEwhdbnO/qu7xNSpTra+++oqkpCRatmzJX/7yl6DDMQaoIZGo6l5gL3Bt7MIxNamsrGT48OFkZGSwfPlyW7jKNBg2q00cSUpKYtq0aWRkZFgSMQ2KJZI4UFRUxL///W+GDh3KyJEjgw7HmBP4uUCWqSeTJk1i7NixHDp0KOhQjAnLaiRxYMGCBWzbto2MjIygQzEmLKuRNFD//ve/mTJlCpWVlbRu3ZoBAwYEHZIx1bJE0kD9/e9/55FHHqGwsDDoUIypledZ5BuKeJlFPlKqioigqvznP/+hbdu2QYdkDFDzLPJWI2lACgoKOPfcc9m2bRsiYknExA1LJA1IZWUlR44coaysLOhQjKkTe2rTAOzevZvs7GwGDBhAQUEBSUmW30188fU3VkSGi8hmEdkiIneFef2HIrLW/fpARBJuluKCggJ69OjBCy+8AGBJxMQl335rRSQZmI8z5UBv4FoR6V3ltM+AIaraF7gXZ43hhHLaaadx3XXXccEFFwQdijER8/PP37nAFlXdqqpHgGeBUaEnqOoHqvq1u5sHdPQxngZlxYoVfPPNN6Snp/PQQw/Rrl27oEMyJmJ+JpIOwPaQ/UL3WHVuwpm24AQiMkFE8kUkf9euXfUYYjB27NjBJZdcwtSpU4MOxZh64Wdja7jhqWE7rYjIxTiJJGz9XlVzcW97cnJy4qvjSxjt27fnqaee4uKLLw46FGPqhZ81kkKgU8h+R2BH1ZNEpC/wCDBKVb/yMZ7APf/883z44YcAjB492hb0No2Gn4lkFdBTRLqJSBowliqLj4tIZ2AhcL2qfuJjLIE7fPgwU6ZM4b777gs6FGPqnW+3NqpaLiK3Aktx1gz+s6p+LCIT3dcXANOBk4AH3Yl6yqvrghvvmjRpwttvv81JJ50UdCjG1Dsba+OzBx98kOLiYmbMmBF0KMZEpaaxNtaz1UeqykcffURxcTHl5eWkpNiP2zRO9pvtk9LSUtLT03n44YepqKiwJGIaNeuP7YNZs2Zx/vnns3fvXpKTk0lLSws6JGN8ZX8mfXD22WezdetWmjVrFnQoxsSE1UjqiaqyYcMGAC6//HJyc3NJTk4OOCpjYsMSST154IEH6N+/P+vXrw86FGNiLuFubRYVFDFn6WZ2lJTSPiudycN6cVX/moYAeXPjjTdSWVnJGWecUQ9RGhNfEqpGsqigiCkL11FUUooCRSWlTFm4jkUFRRFdr6KigkcffZSKigpatmzJL3/5S1sBzySkhEokc5ZuprSs4rhjpWUVzFm6OaLrvfbaa4wfP57FixfXfrIxjVhC3drsKCmt0/HajBgxgnfeeYeLLroomrCMiXsJVSNpn5Vep+PhHDlyhJtvvplPP/0UwJKIMSRYIpk8rBfpqcc/kk1PTWbysF6er7Ft2zZeeOEFli9fXs/RGRO/EurW5ujTmUie2lRWVpKUlMQpp5zC5s2badWqld/hGhM3EiqRgJNM6vq499ChQ4waNYoxY8bw05/+1JKIMVUk1K1NpFJSUsjIyCAjIyPoUIxpkBKuRlIXe/fuJSUlhaZNm7Jo0SLrI2JMNaxGUo2KigqGDx/OmDFjji3sbYwJz2ok1UhOTuaWW24hMzPTkogxtbBEUkVxcTHbtm0jJyeHcePGBR2OMXHBbm2qGD9+PCNHjqS0NLLersYkIquRVDF//ny++OIL0tO993Y1JtFZjQT4/PPPmT17NqpKp06dbEFvY+rIEgnw2GOPMWvWLIqKIptOwJhEZ4kEmD59OgUFBXTs2DHoUIyJSwmbSDZs2MB3v/tddu3aRVJSEl27dg06JGPiVsImkl27dvH555+zZ8+eoEMxJu4l3FObAwcO0KxZM4YMGcLGjRtJTU0NOiRj4l5C1UjWrFlD9+7defXVVwEsiRhTTxIqkXTr1o1LL73UZno3pp4lRCJZs2YN5eXltGjRgmeeeYYuXboEHZIxjUqjTyTbt29n0KBBTJs2LehQjGm0fG1sFZHhwANAMvCIqs6q8rq4r38POAT8SFU/iqbMcAtgzZs3jyuvvDKayxpjauBbjUREkoH5wBVAb+BaEeld5bQrgJ7u1wTgoWjKDF0A6+CWlXz+6SdMWbiOVv2H0bp162gubYypgZ+3NucCW1R1q6oeAZ4FRlU5ZxTwhDrygCwRaRdpgUcXwKosO8yepfP5+p3Ho1oAyxjjjZ+3Nh2A7SH7hcB5Hs7pAHwZepKITMCpsdC5c+dqCzy60FVSahNOHnsfyc1aHXfcGOMPP2sk4aYV0wjOQVVzVTVHVXNqukUJXegq9aSOJDXJOOG4Mab++ZlICoFOIfsdgR0RnONZfSyAZYypOz8TySqgp4h0E5E0YCxQdbXtxcAN4hgI7FXVL6teyKur+ndg5ug+dMhKR4AOWenMHN2nzuvYGGPqxrc2ElUtF5FbgaU4j3//rKofi8hE9/UFwKs4j3634Dz+/XG05UayAJYxJjq+9iNR1VdxkkXosQUh2wrc4mcMxhj/NfqercYY/1kiMcZEzRKJMSZqlkiMMVETp70zfojILuALD6dmA7t9DidaFmP0Gnp80PBj9BpfF1UN2yM07hKJVyKSr6o5QcdRE4sxeg09Pmj4MdZHfHZrY4yJmiUSY0zUGnMiyQ06AA8sxug19Pig4ccYdXyNto3EGBM7jblGYoyJEUskxpioxX0iEZHhIrJZRLaIyF1hXhcRmeu+vlZEzm6AMf7QjW2tiHwgIv0aUnwh550jIhUiMiaW8bll1xqjiAwVkdUi8rGIvNOQ4hORTBF5WUTWuPFFPdK9jvH9WUSKRWR9Na9H9zlR1bj9wpme4FOgO5AGrAF6Vznne8BrOLOxDQT+1QBjPB9o6W5fEcsYvcQXct5bOKO5xzTAn2EWsAHo7O63aWDx/QqY7W63BvYAaTGM8SLgbGB9Na9H9TmJ9xpJzCeY9iNGVf1AVb92d/NwZoprMPG5fga8CBTHMLajvMR4HbBQVbcBqGos4/QSnwLN3SVYmuEkkvJYBaiq77plVieqz0m8J5LqJo+u6zl+qmv5N+H8ZYiVWuMTkQ7A1cACguHlZ3gq0FJElovIhyJyQ8yi8xbfPOB0nKlE1wG3qWplbMLzJKrPia8TG8VAvU0w7SPP5YvIxTiJ5AJfI6pSbJhjVeP7I3CnqlY4f1BjzkuMKcAA4FIgHVghInmq+onfweEtvmHAauASoAfwDxF5T1X3+RybV1F9TuI9kcR8gukIeCpfRPoCjwBXqOpXMYoNvMWXAzzrJpFs4HsiUq6qi2ISoff/592qehA4KCLvAv2AWCQSL/H9GJilToPEFhH5DDgNWBmD+LyI7nMSq8YenxqQUoCtQDe+beQ6o8o53+f4RqSVDTDGzjjz1p7fEH+GVc5/jNg3tnr5GZ4OLHPPzQDWA2c2oPgeAma42ycDRUB2jH+OXam+sTWqz0lc10g0oAmmfYhxOnAS8KD7V79cYzRa1GN8gfISo6puFJHXgbVAJc5a02EfdQYRH3Av8JiIrMP5sN6pqjGbWkBEngGGAtkiUgj8D5AaEl9UnxPrIm+MiVq8P7UxxjQAlkiMMVGzRGKMiZolEmNM1CyRGGOiZokkzonIJBHZKCJP1XDOUBFZEsu4qiMiI4+OjhWRq0Skd8hr94jId2MYy1AROT9W5TVmcd2PxABwM05v2M+CDsQLVV0MLHZ3rwKW4IzaRVWn13d5IpKiqtUNjhsKHAA+qO9yE43VSOKYiCzAGbq+WER+ISLnuvOZFLjfe4V5zxB3zo7V7nnN3eOTRWSVOxfF3dWUd0BEficiH4nIMhFp7R4/S0Ty3Pf+XURauscnicgG9/iz7rEficg8tyYwEpjjxtJDRB4TkTEicoWIPB9S7lARedndvlxEVrgxvCAizcLEuVxEfuPOSXKbiFwpIv9y/71visjJItIVmAj8wi3/QhFpLSIvuj+HVSIyOJr/n4QSyy669uVLt+fPcbtaAy2AFHf7u8CL7vZQYIm7/TIw2N1uhlMrvRxnAmDB+eOyBLgoTFkK/NDdng7Mc7fXAkPc7XuAP7rbO4Am7naW+/1HIe97jJDu9kf33Zi2AU3d4w8B43DG+bwbcvxOYHqYOJcDD4bst+Tbzpfjgd+52zOA/xty3tPABe52Z2Bj0P+/8fJltzaNSybwuIj0xPnQp4Y5533g926bykJVLRSRy3GSSYF7TjOgJ86HNlQl8Jy7/SSwUEQycZLE0RnJHgdecLfXAk+JyCJgkdd/hDpdzl8HrhSRv+GMA7kDGAL0Bt53hxKkASuqucxzIdsdgefc+TXSgOpuA78L9A4Z4dxCRJqr6n6vsScqSySNy73A26p6tVt1X171BFWdJSKv4IyryHMbNwWYqaoP17G82sZXfB9nZq6RwDQROaMO134OuAVnMp5VqrrfnRToH6p6rYf3HwzZ/hPwe1VdLCJDcWoi4SQBg1S1tA5xGqyNpLHJxBlVCs4txAlEpIeqrlPV2UA+zlD2pcB/HW1vEJEOItImzNuTcG49wJmR7J+quhf4WkQudI9fD7wjIklAJ1V9G6c2kYVT0wm1H2hezb9lOc7UgD/h29pFHjBYRE5x48wQkVOreX+o0J/LjTWU/wZw69EdETnLw7UNlkgam98CM0XkfZxRqOH8XETWi8gaoBR4TVXfwGkfWOGOTv0b4T/gB4EzRORDnAl67nGP34jTaLoWOMs9ngw86V6vAPiDqpZUud6zwGS3EbRH6AuqWoHTVnOF+x1V3YWTIJ9xy8rDSYS1mQG8ICLvcfxi2S8DVx9tbAUmATlu4/AGnMZY44GN/jWeicgBVT3hKYkxViMxxkTNaiTGmKhZjcQYEzVLJMaYqFkiMcZEzRKJMSZqlkiMMVH7/0GdgVLmimpwAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
    " + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], "source": [ "# ROC for deterministic forecasts and bin_edges\n", "roc = xs.roc(obs, fct, np.linspace(0, 1, 11), return_results='all_as_metric_dim')\n", @@ -1398,7 +1437,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 37, "metadata": {}, "outputs": [], "source": [ @@ -1417,7 +1456,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ @@ -1427,22 +1466,22 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 39, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[]" + "[]" ] }, - "execution_count": 38, + "execution_count": 39, "metadata": {}, "output_type": "execute_result" }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEWCAYAAACNJFuYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABBx0lEQVR4nO3dd3xV9f3H8dc3e4+bvUMGBMJIEBARVBxVcG+oWlcdtQra1mprbWvtsNU6UNufo1brAAcO3IqLIYhAEkYImWTvRfa49/v74ybXhCQQIMm9ufk8H4/7yL3nnHvP9zDuO99xvl+ltUYIIYToy8HaBRBCCGF7JByEEEIMIOEghBBiAAkHIYQQA0g4CCGEGEDCQQghxAASDmJCUEpdp5TaNNLHCmGvJByEsFFKqbuUUhVKqUal1AtKKdfDHJuilNqhlGrt+ZnSZ991SimjUqq5z+O0MbgEMY5JOAhhg5RSZwP3AmcAsUAc8MAQx7oA7wGvAP7AS8B7Pdt7bdFae/V5fD2KxRd2QMJB2A2l1L1KqTylVJNSKlMpdfFhjtVKqRVKqXylVI1S6mGllMMhxzyilKpXShUopZb02X69Umpfz3nylVK3jMLlXAv8R2u9V2tdDzwIXDfEsacBTsDjWusOrfUqQAGnj0K5xAQh4SDsSR6wCPDF/Fv2K0qpsMMcfzEwB5gNXAjc0GfficB+IBD4B/AfpZTq2VcFnAf4ANcDjymlZg92AqXUQqVUw2EeC4coWzKQ0ed1BhCilAoY4thduv9cOLt6tvdK7QnBbKXU/UoppyHOKwQg4SDsiNb6Ta11mdbapLV+HcgB5h3mLX/XWtdprYuAx4HlffYVaq2f01obMTfThAEhPef5UGudp82+AT7DHEqDlWmT1trvMI+hOr69gMY+r3ufew/j2N7je4/dAEwHgoFLe67z7iHOKwQg4SDsiFLqJ0qp9N7fyjF/IQYe5i3FfZ4XAuF9Xlf0PtFat/Y89eo5zxKl1FalVF3PeZYe4TzHohlzzaRX7/OmYRzbe3wTgNY6X2td0BOau4E/AZeNcHmFnZFwEHZBKRUDPAfcDgRorf2APZjb3ocS1ed5NFA2jPO4AmuBR4CQnvN8NNR5lFKLDhkldOhj0BoHsBeY1ef1LKBSa107xLEz+zR7Aczs2T4YPVR5hegl4SDshSfmL71qMHcaY645HM7dSil/pVQUsBJ4fRjncQFce87T3dNR/aOhDtZabzxklNChj41DvPV/wI1KqWlKKX/gd8CLQxz7NWAEViilXJVSt/ds/xIsNZ2QnudJwP2YRzcJMSQJB2EXtNaZwD+BLUAlMAPYfIS3vQfsANKBD4H/DOM8TcAK4A2gHvgxsO5Yy32Y83yCuSP8K8xNXoXAH3r3K6U+Vkr9tufYTuAi4CdAA+aO9Yt6toN5OOwupVQL5lrO28BfR7rMwr4oWexHTERKKQ0kaq1zrV0WIWyR1ByEEEIMIOEghBBiAGlWEkIIMYDUHIQQQgxgF7fQBwYG6tjYWGsXQwghxpUdO3bUaK2DBttnF+EQGxvL9u3brV0MIYQYV5RShUPtk2YlIYQQA0g4CCGEGEDCQQghxAASDkIIIQaQcBBCCDGAhIMQQogBJByEEEIMIOEghBAjpLiulQ92DVwzqr6lkze3F9NtNPXb3t5l5I3vi2np6B6rIg6bXdwEJ4QQ1tba2c11/91GXnULRpPmwpQIAEwmzYo1aWzMqaG0oY07z5xsec9fPtzHy1sL2ZxXw+NXptB/MT/rkpqDEEKMgAc/yCS/poW4IE/ue2cPRbXmpcef35TPxpwaEoK9WPVFDtsK6gD4bG8FL28tJCHYi/fSy3h7Z6k1iz+AhIMQQhynj3aXs3pbMbeeGs//bpiHUrBiTRo7Cuv5xyf7OSc5lHduW0CUwYM716Sxv6KJX6/dxfQIHz64YyEnTjJw/3t7KKhpsfalWEg4CCHEcSipb+XetbuYFeXHL86aTKS/Bw9dMpP04gaWP7uVYG9XHrp0Bt5uzqxalkpVUwfnP7mJzm4Tq5al4ubsyOPLUnBxcmDF6jQ6u02DnmOw7aNJwkEIIY5Rt9HEnWvSMWl4clkqzo7mr9RzZ4axbG4URq15fFkqfh4uAMyK8uPus6fQaTTxwAXJxAV5ARDm684/Lp3J7tJGHv40q985dpU0sPiRr7nt1Z2M5fo7drHYz5w5c7TMyiqEGGuPfZ7NE1/k8MSyFEsHdC+TSVPV1EGor9uA95U3thHm6z5g+/3v7uHlrYW8dMM8Tp0cRHNHN+eu2kjlwXbau0z86cJkfnJS7IiVXym1Q2s9Z7B9UnMQQohjsK2gjie/zOGS2REDggHAwUENGgzAoMEAcN+5U5kS4s0v38iguqmD37+3h+K6Vv53w4ksnhLEnz/cx77ygyN6HUORcBBCiKPU0NrJnWvSiDZ48KcLp4/Y57o5O7JqeSpN7V1c/n/f8vbOUlackci8SQYevnwWvu7OrFidRlunccTOORQJByGEOApaa+5du5vq5g5WLU/Fy3VkbxebEurN786bxoHaVubG+nP74gQAAr1cefSKWeRUNfPgh5kjes7ByE1wQghxFFZvK+aTvRX8ZkkSMyP9RuUcV58YjcHDhflxBpwcf/gdflFiELecGscz3+RzSmIg50wPG5Xzg9QchBBi2HIqm/jTB3tZlBjITYviRu08SinOnRlGgJfrgH2/PGsKMyN9uWftbsoa2katDBIOQggxDO1dRu5YnYanixP/vGIWDg7WmerCxcmBVctSLcNojabRGXEq4SCEED1MJs2f3s/knbSSAfv+9tE+siqaeOSKWQR7Dz4KaazEBnry4EXT2Xagjqe+zB2Vc0ifgxBC9HhhcwEvbC7A2VGREOTNjEhfANZnVvLSlkJuOHkSi6cEW7mUZpfMjmRLXi3uLqPzO77cBCeEEMDukkYu+fdmFiYEklXRhKuTAx+sWERzezdLnthAuJ87b9+2AFcnR2sX1UJrfVwzuR7uJjipOQghJryWjm5WrEkj0MuVx65MYX9FE8uf28rv391DRc/dyauWp9pUMACjOsW3hIMQYsL7w7q9FNa28NpN8/HzcOHEuABuPz2RVV/kAPCPS2cS3zMP0kQh4SCEmNDeSy/lrR0lrDg9gflxAZbtK05PYF/5QUJ8XLl8TqQVS2gdEg5CiAmrqLaV372zhxNi/FlxRmK/fU6ODjz3k0Gb4ycEGcoqhJiQuowmVqxJAwVPLEvpdyeykHAQQtgxrTX7K5owDXKj2GOfZ5Ne3MDfLplBpL+HFUpn2yQchBB2a833xZz9+Ab+8en+ftu/za3h39/kceWcKM6bGW6l0tk2CQchhF3KqWzigff34uniyDMb8ticWwNAXUsnd76ezqRAT/5wwTQrl9J2STgIIexO33mQPlq5iLhAT+56PZ3a5g5+/VYGDa1drFqWioeLjMkZioSDEMLuPPRxlnkepMtnERPgyZPLZ9PQ2sUFT21m/b4q7lmSxPQIX2sX06ZJOAgh7MoX+yp58dsDXH9yLIuTzPMgTQv34TdLkyhtaGPxlCBuODnWuoUcB6ROJYSwG5UH2/nVmxlMC/PhnnOS+u27bkEsEX7uzI8PGNVpJ+yF1WoOSqkopdRXSql9Sqm9SqmVPdsNSqnPlVI5PT/9rVVGIcT4YTRp7no93TIPkptz/3mQlFL8KDkUHzdnK5VwfLFms1I38Eut9VRgPvBzpdQ04F7gC611IvBFz2shhDisZzbk8W1eLX+8YBoJwRNrHqTRYLVw0FqXa6139jxvAvYBEcCFwEs9h70EXGSVAgohxo20onoe/Sybc2eEccWcKGsXxy7YRIe0UioWSAW+A0K01uVgDhBg0JU1lFI3K6W2K6W2V1dXj1lZhRC25WB7FyvWpBHi48ZfL54xofoT2traaGpqGpXPtnqHtFLKC1gL3Km1Pjjcv1it9bPAs2Be7Gf0SiiEsFVaa+5/dw+l9W28cctJ+HrYV3+CyWSiqamJuro66uvrLT97H+3t7cyYMYNLLrlkxM9t1XBQSjljDoZXtdZv92yuVEqFaa3LlVJhQJX1SiiEsGVv7yzlvfQyfnHWZObEGqxdnGPS3d1NQ0PDgACoq6ujoaEBo9FoOdbBwQE/Pz/8/f2JiIjAYDAQHj46039YLRyUuYrwH2Cf1vrRPrvWAdcCD/X8fM8KxRNC2LiCmhbuf28P8yYZ+PniBGsX57A6OzstX/i9j97XjY2N/Y51cXHB39+f4OBgpkyZgr+/PwaDAX9/f3x9fXFwGJveAGvWHE4GrgF2K6XSe7b9FnMovKGUuhEoAi63TvGEELaqs9vEitVpODs68PiVKTg6WL+foaOjo9+Xf98AOLRfwN3dHYPBQHR0tOXLvzcAPD09baLfxGrhoLXeBAz1J3DGWJZFCDG+PPLZfnaXNvJ/V59AuJ/7mJ23bwDU1tZavvxra2tpaWnpd6yXlxcGg4H4+HjLF39AQAD+/v64ubmNWZmPldU7pIUQ4mhsyK7m2Q35XHViNOdMDx3xz+/q6rJ84R/689AA8Pb2xmAwkJiYSEBAgKUGYDAYcHFxGfGyjSUJByHEqMutaibYx/W4706uae7gF29kMDnEi/vPO/bpto1GY7/f+vsGwMGDB/sd21sDSExMxGAw9AuB8R4AhyPhIIQYVXtKG7nkX98yNcybN29dgIvTsXWomkyaX72ZwcH2Ll756bwB02McSmvNwYMHLV/+fQOgvr4erX8YAe/u7k5AQACxsbGWAOgNAVdX12Mq73gn4SCEGDWtnd2sWJOGq7MDGSWNPPp5NvcuSTryGwfxwuYCvt5fzYMXJpMU6mPZ3t7ebvnyr6mp6RcEXV1dluOcnZ0xGAyEhYWRnJyMwWAgMDAQg8GAh4csE3ooCQchxKh5YF0mBTUtvPrTE3k/o5xnNuSxMCGQhYmBR/U5u4rr+dcn6Zwf60q8LuP993dbwqBvP4BSCj8/PwIDA5k0aZKlBhAQEIC3t7dNjAIaLyQchBCj4v2MMl7fXsztixNYEB9IapQ/3x+o46430vlk5SICvAY217S3t1NTU2OpAdTU1FBVXUNNbS0XumiohM8/Bw8PDwICAkhMTCQwMLBfM5Cj4+Gbm8TwSDgIIUZccV0rv317N6nRfqw8MxEAdxdHnlyeyoVPb+LeNd9xz2nh1NbWUl1dbQmC5uZmy2c4ODhgMBio6nRiX3cIyxZN48SpsQQEBEgz0BiQcBBCjKhuo4mVa9JQmHjgR1Hk7M+y1AZqamq42q0aXdrNq6+aj3dzcyMwMJCEhAQCAgIICgoiMDAQPz8/PtpTycOr07h9cQKX/WiKdS9sgpFwEEIcl66uLmpqaqiurqa6upotmQeIqalhlmMH77y6zXKcr68vgYGBzJtzAh/nNLOj0sgT15/C7LjQQfsCemsfs/vUPsTYkXAQQgxLR0dHvxDofTQ0NFiOUUrRYHTFy9efhbPiLbWAwMDAfvcEzFvUyTmPb+DX7+7n/TuC8HDp/1XUW/sAeGJZKs6ONrG6wIQi4SCE6Kejo4Pq6mqqqqqorq42dwpXVfW7OczR0ZHAwEAiIyNJSUkhKCgIF08/rnllL+4ezrz/84V4ug799WLwdOHxK1O46j/f8cC6TP5+2cx++5/4IoedRQ08uTyVKIP0L1iDhIMQE1RnZ6flt//eIDg0BJycnAgMDCQmJoagoCDLw9/fv9/soFprbn55BzWtXbxz3bzDBkOvBQmB/OzUeP71dR6nTA7i3JlhAGzJq+Wpr3K5/IRIzp81OtNRiyOTcBDCznV3d1NbW0tlZWW/EOjbHOTo6EhQUJAlBIKDgwkKCsLPz29YU0S/8l0Rn2dW8rtzpzI9wnfYZbvrrMl8m1fLvW/vYlaUL54uTtz1ejqTAjz54wXJx3K5YoRIOAhhJ7TW1NfXU1VVRWVlJdXV1VRWVlJbW2uZKsLBwYGAgAAiIiJISUkhODiY4ODgATWBo7G/ook/f5DJqZODuOHkSUf1XmdHB1YtS2Xpqo2sXJOOv4cLtS0dPH/tycOqfYjRI3/6QoxDra2tVFZWWmoDvWHQd7qI3gVjkpKSCAkJITg4mICAgBG9Say9y8gdq3fi7ebMI5fPwuEY1lWIDvDgLxdPZ+WadICjrn2I0SHhIIQNMxqN1NTUWIKg99H3ZjEPDw9CQkKYPXs2wcHBhISEmDuIx2DG0D9/mEl2ZTMv3TCPIO9jn6DuwpQIMssPUt/SedS1DzE6JByEsBHNzc1UVlZSUVFBVVUVFRUV1NTUYDKZgB/6BeLj4y01gZCQELy8vKxS3k/3VvDK1iJuWjSJUycHHffn/WbJ1BEolRgpEg5CjDGTyURtbS0VFRWWMKioqOg3gZy3tzehoaEkJiYSEhJCaGioTc0bVN7Yxj1rdzEjwpe7zz62WVaFbZNwEGIUdXZ29guA3mah7u5uwNxBHBwcTEJCgiUEQkJCjmvuoLzqZjxdnAj1Hd5SlAfbuyiqbR12O7/RpLlzTTqd3SZWLU895vUZhG2TcBBihLS2tlJRUUF5ebklDPqOFHJzcyM0NJQ5c+YQGhpKaGgogYGBI1obKKpt5cKnNuPr7sxHKxfh6374lde6jSau/+/37CyqZ/VN85kfF3DEc/zrq1y+K6jj4ctmMinQc6SKLmyMhIMQx6C5uZny8nLKysosgdDY2GjZ7+PjY1lUpjcIfH19R3U9gS6jiTvWpKGAyoPt/Pbt3Tz149TDnnPVl7nsKKzH38OZu15P56MVi/D3HLoje0dhHY9/kcMFs8K57ITIUbgKYSskHIQ4gqamJsrKyigvL7cEQt/RQgaDgaioKObOnUtYWBihoaFWmVL60c+zyShu4F9XzeZAbQv/+GQ/p2wP5Mq50YMe/11+LU99mcOlsyO5/uRYLv7XZu5Zu4tnrjlh0EBpbOtixep0wv3c+PPF02XhHDsn4SBEH83NzZSVlVnCoG8QKKUIDAwkdtIkwsLCiAgPJzQ01CbWGN6cW8P/fZPH8nlRLJ0Rhsmk2Zxbwx/XZXJCjIGE4P4jmhpaO7nz9XRiAjx54MJkvFyduOecJP784T5e/a6Iq+fH9Dtea8197+ym8mA7b956Ej5uh2+uEuOfhIOYsNra2ixBUFZWRmlpKU1NTZb9gYGBxMfHExYWRnh4OCEhIRhx5JJ/f4t3Fbw6L8omZgutbe7grtfTiQv05P7zpgHg4KB49IoUznl8A3esTuOd2xbg5mzu29Bac8/aXdQ0d/D2z07Gq+dO5BtOnsSGnBoe/CCTubEGpoR6W87x5vYSPthVzt1nTyE12n/sL1KMOQkHMSF0dXVRUVFBaWmpJQjq6uos+w0GAzExMYSHhxMeHk5YWNigN5H99p3d7Cs3T0z3+Ppsqw/j1Fpz91u7aGjt4sXr5/Wb+jrEx41HLp/FjS9t5++fZPGH881zFb22rYhP91by26VJzIj8YYSSg4Pin5fPYskTG7lj9U7W3b4QN2dH8qqb+cO6vSyID+DWU+PH/BqFdUg4CLujtaampobS0lLLo7Ky0nIzmY+PD+Hh4aSkpBAREUFYWBju7u5H/NxP9pTz2ndF3HJqHA0tXfzr6zxOTghkQXzgaF/SkF769gBfZlXxx/OnMS3cZ8D+M6aGcN2CWP67+QCLEgOJ9PfgT+9nsigxkJ8ujBtwfJC3K/+8YhbXvrCNv3y4j9+dN5U7XkvDzdmBx65MwfEYpscQ45OEgxj3WltbKS0tpaSkhJKSEkpLS+no6ADA1dWV8PBwFixYQEREBBEREXh7ex/hEwcqbWjj12/tYmakL788awrdJhPfF9Zx1+vpfLzyFAyHGeEzWjLLDvLXj7I4IymYaxfEDnncvUuS2Jpfy6/e3EWApwvebk7884qh50E6dXIQNy2axHMbC8ipaiKz/CDP/2QOIT7Du29C2AcJBzGumEwmqqqqLEFQUlJCbW0tYO4wDgkJYfr06URGRhIREUFgYOBxj6oxmjR3rUnHaNKsWma+6csFB55cnsrFT3/Lr9/axXM/6T/C59XvCvlmfzWrlqda2vpHUmtnN3es3omfhzP/uGzmYa/RzdmRp36cynlPbqKupZMXr59LsPfhv+jvPjuJrfl1bM2v47oFsZw5LWSkL0HYOAkHYdPa29spKSmhuLiY4uJiSktL6ezsBMwTzkVFRZGSkkJkZCTh4eGjMtncU1/msu1AHY9eMYvYPjd9JYf7cu+SJP70QSYvby3kJyfFApBWVM/v39uL0aR56OOsUVmX4MEPMsmvaeGVG08kwOvIo6USgr159po5VDd1cNqU4CMe7+LkwL+vns3bO0u5+ZSBzU/C/kk4CJuhtaaxsZGioiKKioooLi6mqqoK+KFWMHPmTKKiooiKisLPz2/Ux9p/f6COJ77I5uLUCC6ZPfCmr+tPjmVTbg1//nAfc2MNRPi7s2JNGqE+bixKDOTFbw+wMCFwRH/z/mh3Oau3FfOz0+I5OWH4/R2nHOXkeJH+Hqw4I/FoiyfshOq9tX88mzNnjt6+fbu1iyGOUm8TUW8YFBUVWYaSurq6EhkZaQmCiIiIMb+foLG1i6WrNuLooPhwxUK8hxjbX9vcwTlPbMTP3ZnJId58sreCN26Zz/QIXy56+lsqGtv45M5TRqTNvqS+laVPbGRSkBdv3XqSTQylFeOXUmqH1nrOYPuk5iDGjNFopLy8nMLCQgoLCykqKrJ0HPv4+BATE0N0dDRRUVEEBwcf88pkI0Frzb1v76LyYDtv/WzBkMEAEODlymNXpHDNC9+RU9XML8+azAkxBgCeXJ7K+U9u4q7X03n5xhOHNdqnqb2LZzfkc/X8mH6B0m00ceeadEwaVi1LkWAQo0rCQYya7u5uSktLLWFQXFxsWaksMDCQ5ORkSyCM9rxDR2vN98V8vKeCe85JIiXK74jHL0wM5LdLprKv4iC3LU6wbE8I9uKPF0zjnrW7eWZDHredlnCYT+m9E3kP6zLK+K6gjtU3zbcEypNf5rK9sJ7Hr0whJkAmvBOjy6rhoJR6ATgPqNJaT+/ZZgBeB2KBA8AVWut6a5VRDJ/RaKS0tJSCggJLGPROTR0SEkJqaioxMTHExMTg6Wm7X265VU088P5eFiYEcstRdMbeNMSxV8yJYkN2DY9+ls1JcQGHvcN47c5S1mWUcXJCAJtza3n6q1xWnJHItoI6nvwyh0tmR3BRasRRX5MQR8uqfQ5KqVOAZuB/fcLhH0Cd1vohpdS9gL/W+p7DfY70OViHyWSivLycgoICDhw4QFFRkaVmEBoaSmxsLLGxsURHRw/rJjNb0N5l5KKnN1PV1MEnKxcRPEJj+xvbulj6xEYcHOCjFYsGbabKr27mvCc3MTPSl1d/Op9fvpHOuowynvvJHO5/dw8uTg58sGKRZboLIY6XzfY5aK03KKViD9l8IXBaz/OXgK+Bw4aDGBtaa2pra8nPzyc/P58DBw5Y+gyCg4NJTU21BMJ4CYNDPfRxFlkVTfz3urkjFgwAvu7OrFqewhXPbOV37+7h8StT+jWjdXabWLkmHRenH+5EfvCi6ewsauDGl7bj5KB4+7YFEgxizNjiv7QQrXU5gNa6XCk16KBspdTNwM0A0dGDT0ksjl9LS4slDPLz8zl40DyvkJ+fH9OmTSMuLo7Y2FirrWM8kr7YV8mL3x7g+pNjWZx05HsBjtYJMQZWnpHIo59nc0piEJf2WQ/h4U+z2F3ayDPXnECYrzlYvd2cWbU8laue28ovfjSFmZF+I14mIYZii+EwLFrrZ4FnwdysZOXi2A2j0UhxcTG5ubnk5+dTXl4OmFcxi4uLszz8/e1rZs7Kg+3c/dYupoX5cO+S0ZtM7+eLE9iUW8Pv39vD7Bh/JgV68k12Nc9tLOCa+TGcnRza7/iUKD/Sfv8jWYpTjDlbDIdKpVRYT60hDKiydoHsXWNjIzk5OeTm5lJQUEBnZycODg5ERkayePFiy7TV1hxaOppMJs0v3kinrdPIquWpuDqN/HQXvRwdFI9fmcKSJzayYnUaz1xzAr98I50pId7cd+7UQd8jwSCswRbDYR1wLfBQz8/3rFsc+9NbO8jJySEnJ4fq6moAfH19mTFjBgkJCUyaNMkmFrEZitaa1k4jnoO0wbd0dB/V9mc25LM5t5a/XzpjwKI4oyHcz52/XzqTW1/ZwdJVG2nrNPLqT+ePyhxMQhwraw9lXY258zlQKVUC/AFzKLyhlLoRKAIut14J7UdbW5slDHJzc2lvb8fBwYGYmBhSUlJITEwckUnqxsrdb+3i88xK3r99IdEBPyzJ+fRXuTyxPodXfnoi8yYZLNs/2FXGnWvS+fulM/u19acXN/DPz/Zz7owwrpgTNWblP2d6KFedGM2r3xXx4EXT+y2sI4QtkOkz7Fh9fT1ZWVns37+foqIitNZ4enqSmJjI5MmTiYuLs+nawVDe3lnCL97IQCmYGelnmUZi+4E6rnx2KyatCfNx4+OVp+Dr4UxxnXnKiebObtydHfngjoXEBXnR1N7Fuas2YTRpPlq5CF/3sV36sstoYndpI6lRoz9HlBCDsdmhrGJkaa2prKxk3759ZGVlWSatCw4OZuHChUyePJmIiIhx/UV0oKaF+9/dw7xYA9ecFMMdq9N47PNsbjk1npVr0onwc+ehS2dw7QvbuPftXaxansrKNWkAvHnLSfz0f9tZsSaNt392Mve/u4eS+lbeuOWkMQ8GAGdHB2bLkpvCRkk4jHNaa0pLS8nMzCQrK4v6evPN5NHR0fzoRz9iypQpGAyGI3zK+NDZbWLFmjScHB14bFkKEX7ubM6t4d/f5PFtXi2VB9t589aTSI3251c/msLfPs7iyme2sLOogSeXpzIn1sA/Lp3JzS/vYNmz5u2/OGsyc2Lt489HiJEk4TAOaa0pKSkhMzOTzMxMDh48iIODA3FxcSxcuJApU6bY9PQUAB3dRm57ZSdzJxn6rUustea+d/eggD9fNL1fLeefn+9nV0kj/3f1bCL8zPcC/P78aXx/oI704gZ+fc4Uy9QUNy2KY1NuDRtzarj8hEjOnxUOwI+SQ7l6fjSvbC1i3iQDP198+LmOhJioJBzGCa01FRUV7Nmzh71799LY2IijoyPx8fGcfvrpTJkyBTe38bOM498/3s8XWVV8ub+KGRG+lnUJXvmuiNe+KwJgWrgPV50YA8DGnGqe+SafH58YzTnTwyyf4+HixPPXzuXLrCqu77NUpkPPkNE3d5RwzfyYfuf+3bnTiPT34JLUCFkTWYghSIe0jauvr2f37t3s3r2bmpoaSw0hOTmZpKSkcRUIvb7KquL6F79n+bwothXU0dTezccrF1HT3MkFT21iflwAJq35/kAd79++EH9PF5b0rJew7vaFuLvIkE8hRoJ0SI8z7e3t7N27l4yMDIqLiwFzH8K5557LtGnT8PDwOMIn2K6qg+386s0MkkK9+cP5yeRXt3DR05v51ZsZlDa04e3mzCOXz0KjWfL4Ru5YnUaIjxuNbV3874Z5EgxCjBEJBxuhtaagoIC0tDT27duH0WgkMDCQM844gxkzZuDr62vtIh43k0nzyzczaOnsZs1y801f08J9+M3SJB54PxOAl26YR5C3eXjtI1fM4vr/fk9WRRMPXJDM1DAfaxZfiAlFwsHKDh48SFpaGmlpaTQ2NuLm5sbs2bNJSUkhLCxsXA87PdRzG/PZmFPDXy+eQWLIDzd9XbcglgM1LUT6e3Bqn3WOF08J5rdLkyhraOcnJ8UM9pFCiFEi4WAFWmtyc3PZsWMH2dnZaK2Ji4vjzDPPJCkpCScn+/tryShu4OFP97NkeijL5/W/E1kpxQMXTh/0fTefEj/odiHE6LK/byEb1tbWRlpaGtu3b6e+vh5PT09OPvlkZs+ebXeznPbV3NHNijVpBHu78tAlM+2qNiSEvZJwGAM1NTV89913ZGRk0NXVRXR0NKeffjpTp07F0dH+O1h//+4eiutaWXPzSfh6jP2dyEKIoyfhMEq01hQVFbF582ZycnJwdHRkxowZnHjiiYSGhh75A+zEO2klvJ1WysozEvtNhCeEsG0SDiNMa012djabNm2ipKQEDw8PTj31VObOnWvzdy2PtMLaFn73zh7mxvpzx+lyJ7IQ44mEwwjRWrN//36++eYbKioq8PPzY8mSJaSmpuLsPPGaUjq7TaxYnWZe3GZZKk6OsmCNEOOJhMNx0lqTl5fHF198QUVFBQaDgQsvvJAZM2ZMiP6EoTz6eTYZJY38+6of5kESQowfhw0HZR5WEqm1Lh6j8owrJSUlrF+/nsLCQvz8/LjooouYMWOG3S6nOVybcmr4v2/yWD4viiUzwo78BiGEzTlsOGittVLqXeCEsSnO+HDw4EHWr1/P7t278fT0ZOnSpcyePXtC1xR61TZ3cNcb6SQEe/H785KtXRwhxDEaTrPSVqXUXK3196NeGhtnNBrZvHkzmzZtwmQysWjRIhYuXIiLi4u1i2YTtNbc/dYuGtu6eOl6mQdJiPFsOOGwGLhFKVUItAAKc6Vi5qiWzMYUFxfz/vvvU11dzdSpUznrrLPs+sa1Y/Hitwf4MquKP54/jWnhMg+SEOPZcMJhyaiXwoZ1dXWxfv16tm3bho+PD8uXL2fy5MnWLpbNySw7yN8+yuKMpGCu7bOughBifDpiOGitCwGUUsHA+Fs84DhUVlaydu1aqqurmTdvHqeffjqurq7WLpbNae3s5o7VO/HzcOYfl8n0GELYgyOGg1LqAuCfQDhQBcQA+wC77W3UWrN9+3Y+/fRT3NzcuPrqq4mPlwnghvLgB5nk17Twyo0nEuAl4SmEPRhOs9KDwHxgvdY6VSm1GFg+usWyHpPJxEcffcSOHTtISEjgoosumnB3Nh+Nj3aXs3pbMT87Ld6y1KcQYvwbzoD8Lq11LeCglHLQWn8FpIxusayjo6OD1157jR07drBw4UJ+/OMf23wwdBlN/O2jfewsqh+1c+wta+RP72fS3mXst72kvpV71+5iVpQfvzhL+mGEsCfDqTk0KKW8gI3Aq0qpKqB7dIs19lpbW3nppZeorq7m/PPPZ/bs2dYu0rA89nk2z2zI5+20Uj5ZuWjEm3UOtndxy8s7KKlvo8to4sGLzOsudBtN3LkmHZOGVctScJbpMYSwK8P5H70B8ANWAp8AecD5o1imMdfZ2clrr71GXV0dV1111bgJhm9za/j3N3ksnhJEY1sXv3ozA631iH2+1pr73tlDeWM7Z04N4eWthXy2twKAVV/msr2wnj9fNJ2YANuuXQkhjt5wwkEBnwJfA17A6z3NTHbBZDKxdu1aysrKuPTSS8dNx3NdSyd3vp7OpEBPnr5qNr87dypf7a/mv5sPjNg53tpRwvsZZfzirMk8fVUq0yN8+PXaXbyXXspTX+ZwyewILkqNGLHzCSFsxxHDQWv9gNY6Gfg55hFL3yil1o96ycaA1poPPviA7Oxsli5dSlJSkrWLNCxaa379VgYNrV08uTwVDxcnrpkfw5lTQ3jo4yz2lDYOeE9aUT1ZFQcHbM+vbmZbQd2g2/+wbi8nxQVw66nxuDo5smpZKp3dJlauSSfa4MGfhljaUwgx/h1NQ3EVUAHUAsGjU5yxlZ+fT1paGosWLWLOnDnWLs6w/W9LIev3VXHvkiSSw30B8zrM/7hsJv6ezqxYk0Zr5w/dQjmVTSx/bivLnt1KeWObZXttcwfLnt3Kj5/bSkZxg2V7R7eRO1an4erkwGNXpuDoYL5vIS7Ii79ePINAL1dWLU/Fy1Um9RXCXh0xHJRSP1NKfQ18AQQCN9nL1Bnx8fFcffXVLF682NpFGbZ95Qf5y0f7OD0pmOtPju23z+DpwmNXplBQ08ID6zIBaO8yf9F7uDjR2W3uRDaaNFpr7lm7i4bWLgyeLqxYk0ZzhzlQHv5kP3vLDvLwZbMI9e1/3+NFqRF8f98ZzIz0G4vLFUJYyXB+9YsB7tRap49yWaxivPQxALR1mr/ofd2deXiIO5EXxAdy22nxPP1VHosmB7L9QD1ZFU3897q51LZ08qs3M/j317l4uzmzfl8Vfzh/Gsnhvix7dgu/f28PF8wK5/lNBVx7UgxnTgsZtBxyB7QQ9m8402fcOxYFsZYN2dUsTAjEwWF4X3j1LZ14uTkNGLrZ3NGNAjwPaWrp7DbR2tmNn0f/mVtNJk1tSydB3gOHnlY1tRPsPXCmkj99kEledTMv33D4O5HvPHMym3NrufvNXbR1Gblx4SQWJwWjtWZDdjWPrc/BUSlOTwrmugWxKKVYcUYij6/P4ZM9FSSFevObpVOH9echhLBPE3pw+sacan7ywjZe2FwwrOOL61o59eGvuO6/2zCafhgy2tTexXmrNnL+U5to6fihrd9o0lzzn+847ZGvKWto6/dZ9769iwUPfUF6n7Z+gH9/nce8v3zB+xll/bZ/vLuc1duKuPmUOBYmHv5OZGdHB55cnoqTgyI53IdfnzMFMP/G/+eLpxPu54afR//ax+2LE5gXa8Bo0jy5PBU3Z5luW4iJTI3kuHhrmTNnjt6+fftRv09rzS0v7+Cr/VW8c9vJTI/wHfLYbqOJK57Zwu7SRrqMml+fM4XbTktAa82dr6fzfkYZGrhsdiQPXz4LgCfW5/DY+mycHRWp0f6svmk+jg6K99JLWbkmHWdHRZivOx+uWIi3mzNpRfVc9n9bcFDg5uTIRysXEWXwoLShjSWPb2BSoCdv/WzBsG84K2tow8fdeUDHcUNrJ11GPaDW0tZppKa5gyiDx9H9QQohxiWl1A6t9aCjcWy25qCUOkcptV8plauUGpWmLaUUf790JgGertyxOq3fb/2HeuKLHHYWNfDoFSmcOyOMRz/LJr24gbd3lvJeehl3njmZn5+WwJs99wZ8f6COJ77I5uLUCB66ZCbbCup4+qtciuta+d07e5gd7cf/bjiRkvpW7n93D03tXaxYk0aojxvv3HYyACvXpNHRbeSu3juRl6ce1Z3I4X7ug44o8vNwGbQ5y93FUYJBCAHYaM1BKeUIZANnASXA98ByrXXmYMcfa82h19b8WpY/t7Xfb/19bcmr5cfP/7C/sa2LpU9sBMy/hSdH+LL6pvmYtObKZ7aQU9mMt5sTTo4OfLhiIV6uTtz1ejrv7yonNsCDqqYOPlphrhX01i6mhHiTW93MG7fM54QYA+syylixOo0pId7sr2zisStncXFq5DFfoxBCHGo81hzmAbla63ytdSewBrhwtE42Py6AOxabf+t/L7203776lk7uej2dSQGe/PEC8yzlvu7OrFqeQsXBdpwcHXi8514AZ0cHnliWCkBVUwerlqfi7eaMUooHL5pOhJ87edUt/PXiGZbf0G8/PYF5kwzsr2zizjMSOSHGAMAFs8K5/IRI9lc2cUlqhASDEGJM2WrN4TLgHK31T3teXwOcqLW+vc8xNwM3A0RHR59QWFh4XOfsNpq48tmtZFc0Wdr6tdbc/PIOvh6iT+Kr/VUEeroyI7L/9vTiBhrbujh1clC/7QU1LewubeSCWeH9ttc0d7A+s5LL50RZbjgD8yI676aVcWFK+IBRUEIIcbwOV3Ow1XC4HDj7kHCYp7W+Y7Djj7dZqVdxXStLV20kIdiLN245iTXbirj/vb387typ/HRR3HF/vhBC2JLx2KxUAkT1eR0JlA1x7IiJMnjwt0tmkFbUwN1vZvDnD/dx6uQgbjh50mifWgghbIqthsP3QKJSapJSygVYBqwbixOfNzOcK+dE8W56Gd5uzjxy+axh3yAnhBD2wiYbsrXW3Uqp2zFPFe4IvKC13jtW5//DBdMwas0Vc6IGHfIphBD2zibDAUBr/RHwkTXO7eHixCODDGkVQoiJwlablYQQQljRhA6HgwcP8sorr9DQ0GDtogghhE2Z0OHQ3t5OaWkpr7zyCq2trdYujhBC2IwJHQ7BwcEsW7aMhoYGVq9eTVdXl7WLJIQQNmFChwNATEwMl1xyCSUlJaxduxaTyWTtIgkhhNVN+HAAmDZtGkuWLGH//v288cYbdHZ2WrtIQghhVRIOPebNm8eSJUvIzs7mxRdfpKmpydpFEkIIq5Fw6GPevHksW7aMmpoann/+ecrLy61dJCGEsAoJh0NMnjyZ66+/Hq01zz//PFu2bMEWJycUQojRJOEwiLCwMG699VYSExP57LPPeO2112hubrZ2sYQQYsxIOAzBw8ODK6+8kqVLl1JQUMDTTz9NWlqa1CKEEBOChMNhKKWYO3cut956K8HBwaxbt46XX36Z2tpaaxdNCCFGlU0u9nO0Rmqxn8PRWrNjxw7Wr19PV1cXJ554Iqeccgpubm6jel4hhBgth1vsx2ZnZbU1SinmzJlDUlISX3zxBVu2bGHXrl2cdtpppKam4ujoaO0iCiHEiJGawzEqKyvj008/paioCIPBwOLFi0lOTkYpWRhICDE+jLs1pI+WNcIBzE1N2dnZfPnll1RVVRESEsKpp55KUlKShIQQwuZJOIwyk8nEnj17+Oabb6irqyM4OJhFixYxbdo0HBykz18IYZskHMZIb0hs2LCB2tpa/P39WbBgAbNmzcLZ2dnaxRNCiH4kHMaYyWRi//79bN68mdLSUjw8PJgzZw5z587Fy8vL2sUTQghAwsFqtNYUFhayZcsWsrOzcXR0ZPr06cydO5eIiAhrF08IMcHJUFYrUUoRGxtLbGwstbW1fPfdd2RkZJCRkUF4eDhz584lOTlZmpyEEDZHag5jrKOjg4yMDLZv3051dTWurq7MnDmTE044gZCQEGsXTwgxgUizkg3SWlNUVMTOnTvZu3cvRqOR8PBwUlJSmDFjhtx5LYQYdRIONq6trY2MjAzS09OprKzEycmJKVOmMGvWLOLj42U4rBBiVEg4jBNaayoqKkhLS2PPnj20tbXh6enJ9OnTmTFjBuHh4XJznRBixEg4jENGo5GcnBx27dpFdnY2RqORgIAAkpOTmT59OkFBQdYuohBinJNwGOfa29vJzMxk9+7dHDhwAIDg4GCSk5NJTk4mICDAugUUQoxLEg52pKmpiczMTPbu3UtxcTEAISEhTJ06lWnTpkmNQggxbBIOdurgwYNkZmaSmZlpCYrAwECSkpJISkqSPgohxGFJOEwATU1N7Nu3j3379lFYWIjWGh8fHyZPnkxSUhIxMTE4Ock9j0KIH0g4TDCtra3k5OSQlZVFXl4eXV1duLi4kJCQwOTJk0lISMDT09PaxRRCWJlMnzHBeHh4MGvWLGbNmkVXVxcFBQVkZWWRk5NDZmYmAJGRkSQmJjJ58mRCQkKk+UkI0Y/UHCYQrTXl5eVkZ2eTk5NDWVkZAN7e3sTHx5OYmEhcXJzcnS3EBGFzzUpKqcuBPwJTgXla6+199v0GuBEwAiu01p8e6fMkHI5Nc3Mzubm55OTkkJeXR0dHB0opoqKiiI+PJz4+nrCwMLlDWwg7ZYvhMBUwAc8Av+oNB6XUNGA1MA8IB9YDk7XWxsN9noTD8TOZTJSUlJCbm0tubi7l5eUAuLu7ExcXZ3n4+flZt6BCiBFjc30OWut9wGDt3BcCa7TWHUCBUioXc1BsGdsSTjwODg5ER0cTHR3N6aefTktLC3l5eeTn55Ofn8/evXsBMBgMlqCIjY3F3d3dyiUXQowGW+uQjgC29nld0rNtAKXUzcDNANHR0aNfsgnG09OTmTNnMnPmTLTWVFdXW4Kid8pxgLCwMCZNmsSkSZOIjo7GxcXFyiUXQoyEUQsHpdR6IHSQXfdprd8b6m2DbBu03Utr/SzwLJiblY6pkGJYlFIEBwcTHBzM/PnzMRqNlJaWkp+fT0FBAVu3buXbb7/FwcGBiIgIywJHUVFRspCREOPUqIWD1vrMY3hbCRDV53UkUDYyJRIjxdHR0dIEddppp9HZ2UlxcTEFBQUcOHCATZs2sXHjRhwdHYmIiCAmJobY2FgiIyOlZiHEOGFrzUrrgNeUUo9i7pBOBLZZt0jiSFxcXCyjm8C82l1hYSGFhYX9wsLBwYHw8HCio6OJiYkhOjpahs0KYaOsNVrpYuBJIAhoANK11mf37LsPuAHoBu7UWn98pM+T0Uq2raOjg+LiYg4cOEBhYSFlZWWYTCbAPGlgby0kJiYGb29vK5dWiInD5oayjjQJh/Glq6uLkpISioqKKCoqori4mK6uLgD8/PyIjo4mKiqK6OhogoKC5O5tIUaJzQ1lFRObs7OzZYQTmBc2qqiooLi4mKKiIvLy8ti1axcArq6uREVFWR4RERHSbyHEGJBwEFbX23EdERHB/Pnz0VpTX19vqVUUFxeTm5sLmEdOhYSEEBkZSVRUFJGRkfj7+0vtQogRJs1KYlxoa2ujpKSk36OzsxMwTzQYGRlJRESE5aerq6uVSyyE7ZNmJTHuubu7k5iYSGJiImCe7qO6urpfWGRnZ1uODwoKstRGIiMjCQ4OljmihDgKUnMQdqO9vZ3S0lJKSkosP9va2gBwcnIiPDyc8PBwS2j4+flJc5SY0GS0kpiQevsuSktLLY/y8nKMRvM8ju7u7v0CIzw8XIbSiglFmpXEhKSUwmAwYDAYmDFjBmAeGVVVVUVpaSllZWWUlpayadMmen9J8vb2tgRGWFgY4eHhsmqemJAkHMSE4ujoSFhYGGFhYZZtnZ2dVFRUUFZWZnns37/fst/Hx8cSFhIYYqKQcBATnouLi+Uu7V7t7e2WwCgvL6esrIysrCzLfh8fH8LCwggNDbWEhre3t/RhCLsh4SDEINzc3Cyzy/bqGxgVFRWUl5f3q2F4enoSGhpqCYzQ0FAMBoMEhhiXJByEGKbBAqO3Sao3LMrLy9myZYtl7igXFxdLYPQ+goKCcHKS/3rCtsm/UCGOw2BNUt3d3VRXV1NeXm4JjrS0NMv8UQ4ODgQFBRESEkJoaKjlp4eHh7UuQ4gBJByEGGFOTk4DOr211tTV1VnCorKykoKCAsscUmAeKRUSEtIvNAICAuTmPWEVEg5CjAGlFAEBAQQEBJCcnGzZ3tLSQmVlpSUwKioqyM/PtzRLOTk5WWoZfR9SyxCjTcJBCCvy9PQkLi6OuLg4y7bu7m5qamqorKy0PLKzs0lPT7cc4+XlRUhICMHBwZaf0pchRpL8SxLCxjg5OVk6r/tqbm62hEVVVRWVlZVs27bNcsd3b+2kd73v3uCQWWvFsZBwEGKc8PLywsvLy7IcK5gnIKytrbWERVVVFeXl5WRmZlqOcXZ2JigoiKCgoH7BIfdliMORcBBiHOsd+RQUFNSvL6Ozs5Pq6mqqqqosj7y8PDIyMizHuLq6Wpqj+gaHp6enhIaQcBDCHrm4uFhmn+2rtbXVEhqVlZVUV1ezb98+du7caTnG3d3dEhi9oREUFCShMcFIOAgxgXh4eBATE0NMTIxlm9aalpYWqqqqLMFRXV3N3r17aW9vtxzXGxqBgYH9wkOap+yThIMQE5xSytKf0XfUlNaa5ubmfoFRU1NDZmZmv9BwdXXtFxq9P2W9jPFNwkEIMSilFN7e3nh7ew8IjZaWFmpqaizBUVNTQ05OTr/htk5OTgQEBFgCozc0DAaDDLkdB+RvSAhxVPrWNPrOMwXmtb57axi9P0tKStizZ0+/9/v7+1sCo+/D3d19jK9GDEXCQQgxYtzd3QfMNQXQ1dVFbW2tJTB6H3l5eZb7NMDcJ9I3LAICAggMDMTPz0+mERljEg5CiFHn7Ow86I19JpOJhoaGfoFRW1tLVlYWra2tluMcHR0xGAyWsOj7U2obo0PCQQhhNQ4ODpalXCdPntxvX2trK7W1tf1Co7q6muzsbMvcU/BDbaN37qreh8FgwNHRcawvyW5IOAghbJKHhwceHh5ERUX12240Gqmvr7cER21tLbW1tWRnZ9PS0mI5TimFn5/fgNAICAjAx8dHRlIdgYSDEGJccXR0tPRJTJkypd++9vb2fqFRV1dHTU0NhYWFlvU04IeRVL01jL7PPTw8JDiQcBBC2BE3N7dB7wzXWtPU1GSpZfQ+KisrycrK6tdM5ebmZgmM3iav3ucTqX9DwkEIYfeUUvj4+ODj48OkSZP67evtFO8bGnV1dRQVFbF79+5+x7q7u/cLjb7h4ebmNpaXNOokHIQQE1rfTvHExMR++7q7uy39G3V1ddTW1lJfX8+BAwf6reIH/YPD39+/X3CMxxqHhIMQQgyhdyW+oKCgAfu6urqor6+nrq6u32Ow4Ohtqjo0OAwGg81OaCjhIIQQx8DZ2dkyzfmh+tY4+gZIaWkpe/fuRWvd73P6hkbf8PDx8bHazX9WCQel1MPA+UAnkAdcr7Vu6Nn3G+BGwAis0Fp/ao0yCiHEsTpcjcNoNNLQ0EBdXV2/AOmdn6rvHeMODg74+flZQqNvgPj7++Ps7Dx61zBqn3x4nwO/0Vp3K6X+DvwGuEcpNQ1YBiQD4cB6pdRkrbXxMJ8lhBDjhqOjo2Xo7KFMJhNNTU2W4Oj7s7i4mI6Ojn7He3l5MX36dM4+++wRL6dVwkFr/Vmfl1uBy3qeXwis0Vp3AAVKqVxgHrBljIsohBBjzsHBAV9fX3x9fQeMqtJa09bW1i806uvr8fX1HZWy2EKfww3A6z3PIzCHRa+Snm1CCDGhKaUsd40feh/HaBi1cFBKrQdCB9l1n9b6vZ5j7gO6gVd73zbI8XqQbSilbgZuBgbMACmEEOL4jFo4aK3PPNx+pdS1wHnAGfqHrvsSoO9EKpFA2RCf/yzwLMCcOXMGDRAhhBDHxipjpJRS5wD3ABdorVv77FoHLFNKuSqlJgGJwDZrlFEIISYya/U5PAW4Ap/33PyxVWt9q9Z6r1LqDSATc3PTz2WkkhBCjD1rjVZKOMy+vwB/GcPiCCGEOISsuyeEEGIACQchhBADSDgIIYQYQPWdAGq8UkpVA4XH8RGBQM0IFWe8mIjXDBPzuuWaJ46jve4YrfXACaCwk3A4Xkqp7VrrOdYux1iaiNcME/O65ZonjpG8bmlWEkIIMYCEgxBCiAEkHMyetXYBrGAiXjNMzOuWa544Ruy6pc9BCCHEAFJzEEIIMYCEgxBCiAEmdDgopc5RSu1XSuUqpe61dnlGg1IqSin1lVJqn1Jqr1JqZc92g1Lqc6VUTs9Pf2uXdTQopRyVUmlKqQ96Xtv1dSul/JRSbymlsnr+zk+y92sGUErd1fPve49SarVSys0er1sp9YJSqkoptafPtiGvUyn1m57vt/1KqaNaS3TChoNSyhF4GlgCTAOW96xhbW+6gV9qracC84Gf91znvcAXWutE4Iue1/ZoJbCvz2t7v+4ngE+01knALMzXbtfXrJSKAFYAc7TW0wFHzGvR2+N1vwicc8i2Qa+z5//5MiC55z3/6vneG5YJGw6Y16bO1Vrna607gTWY17C2K1rrcq31zp7nTZi/LCIwX+tLPYe9BFxklQKOIqVUJHAu8HyfzXZ73UopH+AU4D8AWutOrXUDdnzNfTgB7kopJ8AD8yJhdnfdWusNQN0hm4e6zguBNVrrDq11AZCL+XtvWCZyOEQAxX1e2/161UqpWCAV+A4I0VqXgzlAgGArFm20PA78GjD12WbP1x0HVAP/7WlKe14p5Yl9XzNa61LgEaAIKAcatdafYefX3cdQ13lc33ETORyGvV61PVBKeQFrgTu11getXZ7RppQ6D6jSWu+wdlnGkBMwG/i31joVaME+mlIOq6eN/UJgEhAOeCqlrrZuqWzCcX3HTeRwGPZ61eOdUsoZczC8qrV+u2dzpVIqrGd/GFBlrfKNkpOBC5RSBzA3GZ6ulHoF+77uEqBEa/1dz+u3MIeFPV8zwJlAgda6WmvdBbwNLMD+r7vXUNd5XN9xEzkcvgcSlVKTlFIumDtu1lm5TCNOmddh/Q+wT2v9aJ9d64Bre55fC7w31mUbTVrr32itI7XWsZj/br/UWl+NHV+31roCKFZKTenZdAbmJXft9pp7FAHzlVIePf/ez8Dct2bv191rqOtcByxTSrkqpSYBicC2YX+q1nrCPoClQDaQB9xn7fKM0jUuxFyV3AWk9zyWAgGYRzbk9Pw0WLuso/hncBrwQc9zu75uIAXY3vP3/S7gb+/X3HPdDwBZwB7gZcxr1NvddQOrMferdGGuGdx4uOsE7uv5ftsPLDmac8n0GUIIIQaYyM1KQgghhiDhIIQQYgAJByGEEANIOAghhBhAwkEIIcQAEg5CHIOe2U9v63kerpR6y9plEmIkyVBWIY5BzzxVH2jzLKBC2B0naxdAiHHqISBeKZWO+eajqVrr6Uqp6zDPiukITAf+CbgA1wAdwFKtdZ1SKh7zlPFBQCtwk9Y6a6wvQoihSLOSEMfmXiBPa50C3H3IvunAjzFPj/wXoFWbJ8LbAvyk55hngTu01icAvwL+NRaFFmK4pOYgxMj7SpvXzmhSSjUC7/ds3w3M7JkhdwHwpnkqIMA83YMQNkPCQYiR19HnuanPaxPm/3MOQENPrUMImyTNSkIcmybA+1jeqM3raRQopS4H88y5SqlZI1k4IY6XhIMQx0BrXQts7lno/eFj+IirgBuVUhnAXuxwiVoxvslQViGEEANIzUEIIcQAEg5CCCEGkHAQQggxgISDEEKIASQchBBCDCDhIIQQYgAJByGEEAP8PzIgaIBXQT9BAAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEWCAYAAACNJFuYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABBx0lEQVR4nO3dd3xV9f3H8dc3e4+bvUMGBMJIEBARVBxVcG+oWlcdtQra1mprbWvtsNU6UNufo1brAAcO3IqLIYhAEkYImWTvRfa49/v74ybXhCQQIMm9ufk8H4/7yL3nnHvP9zDuO99xvl+ltUYIIYToy8HaBRBCCGF7JByEEEIMIOEghBBiAAkHIYQQA0g4CCGEGEDCQQghxAASDmJCUEpdp5TaNNLHCmGvJByEsFFKqbuUUhVKqUal1AtKKdfDHJuilNqhlGrt+ZnSZ991SimjUqq5z+O0MbgEMY5JOAhhg5RSZwP3AmcAsUAc8MAQx7oA7wGvAP7AS8B7Pdt7bdFae/V5fD2KxRd2QMJB2A2l1L1KqTylVJNSKlMpdfFhjtVKqRVKqXylVI1S6mGllMMhxzyilKpXShUopZb02X69Umpfz3nylVK3jMLlXAv8R2u9V2tdDzwIXDfEsacBTsDjWusOrfUqQAGnj0K5xAQh4SDsSR6wCPDF/Fv2K0qpsMMcfzEwB5gNXAjc0GfficB+IBD4B/AfpZTq2VcFnAf4ANcDjymlZg92AqXUQqVUw2EeC4coWzKQ0ed1BhCilAoY4thduv9cOLt6tvdK7QnBbKXU/UoppyHOKwQg4SDsiNb6Ta11mdbapLV+HcgB5h3mLX/XWtdprYuAx4HlffYVaq2f01obMTfThAEhPef5UGudp82+AT7DHEqDlWmT1trvMI+hOr69gMY+r3ufew/j2N7je4/dAEwHgoFLe67z7iHOKwQg4SDsiFLqJ0qp9N7fyjF/IQYe5i3FfZ4XAuF9Xlf0PtFat/Y89eo5zxKl1FalVF3PeZYe4TzHohlzzaRX7/OmYRzbe3wTgNY6X2td0BOau4E/AZeNcHmFnZFwEHZBKRUDPAfcDgRorf2APZjb3ocS1ed5NFA2jPO4AmuBR4CQnvN8NNR5lFKLDhkldOhj0BoHsBeY1ef1LKBSa107xLEz+zR7Aczs2T4YPVR5hegl4SDshSfmL71qMHcaY645HM7dSil/pVQUsBJ4fRjncQFce87T3dNR/aOhDtZabzxklNChj41DvPV/wI1KqWlKKX/gd8CLQxz7NWAEViilXJVSt/ds/xIsNZ2QnudJwP2YRzcJMSQJB2EXtNaZwD+BLUAlMAPYfIS3vQfsANKBD4H/DOM8TcAK4A2gHvgxsO5Yy32Y83yCuSP8K8xNXoXAH3r3K6U+Vkr9tufYTuAi4CdAA+aO9Yt6toN5OOwupVQL5lrO28BfR7rMwr4oWexHTERKKQ0kaq1zrV0WIWyR1ByEEEIMIOEghBBiAGlWEkIIMYDUHIQQQgxgF7fQBwYG6tjYWGsXQwghxpUdO3bUaK2DBttnF+EQGxvL9u3brV0MIYQYV5RShUPtk2YlIYQQA0g4CCGEGEDCQQghxAASDkIIIQaQcBBCCDGAhIMQQogBJByEEEIMIOEghBAjpLiulQ92DVwzqr6lkze3F9NtNPXb3t5l5I3vi2np6B6rIg6bXdwEJ4QQ1tba2c11/91GXnULRpPmwpQIAEwmzYo1aWzMqaG0oY07z5xsec9fPtzHy1sL2ZxXw+NXptB/MT/rkpqDEEKMgAc/yCS/poW4IE/ue2cPRbXmpcef35TPxpwaEoK9WPVFDtsK6gD4bG8FL28tJCHYi/fSy3h7Z6k1iz+AhIMQQhynj3aXs3pbMbeeGs//bpiHUrBiTRo7Cuv5xyf7OSc5lHduW0CUwYM716Sxv6KJX6/dxfQIHz64YyEnTjJw/3t7KKhpsfalWEg4CCHEcSipb+XetbuYFeXHL86aTKS/Bw9dMpP04gaWP7uVYG9XHrp0Bt5uzqxalkpVUwfnP7mJzm4Tq5al4ubsyOPLUnBxcmDF6jQ6u02DnmOw7aNJwkEIIY5Rt9HEnWvSMWl4clkqzo7mr9RzZ4axbG4URq15fFkqfh4uAMyK8uPus6fQaTTxwAXJxAV5ARDm684/Lp3J7tJGHv40q985dpU0sPiRr7nt1Z2M5fo7drHYz5w5c7TMyiqEGGuPfZ7NE1/k8MSyFEsHdC+TSVPV1EGor9uA95U3thHm6z5g+/3v7uHlrYW8dMM8Tp0cRHNHN+eu2kjlwXbau0z86cJkfnJS7IiVXym1Q2s9Z7B9UnMQQohjsK2gjie/zOGS2REDggHAwUENGgzAoMEAcN+5U5kS4s0v38iguqmD37+3h+K6Vv53w4ksnhLEnz/cx77ygyN6HUORcBBCiKPU0NrJnWvSiDZ48KcLp4/Y57o5O7JqeSpN7V1c/n/f8vbOUlackci8SQYevnwWvu7OrFidRlunccTOORQJByGEOApaa+5du5vq5g5WLU/Fy3VkbxebEurN786bxoHaVubG+nP74gQAAr1cefSKWeRUNfPgh5kjes7ByE1wQghxFFZvK+aTvRX8ZkkSMyP9RuUcV58YjcHDhflxBpwcf/gdflFiELecGscz3+RzSmIg50wPG5Xzg9QchBBi2HIqm/jTB3tZlBjITYviRu08SinOnRlGgJfrgH2/PGsKMyN9uWftbsoa2katDBIOQggxDO1dRu5YnYanixP/vGIWDg7WmerCxcmBVctSLcNojabRGXEq4SCEED1MJs2f3s/knbSSAfv+9tE+siqaeOSKWQR7Dz4KaazEBnry4EXT2Xagjqe+zB2Vc0ifgxBC9HhhcwEvbC7A2VGREOTNjEhfANZnVvLSlkJuOHkSi6cEW7mUZpfMjmRLXi3uLqPzO77cBCeEEMDukkYu+fdmFiYEklXRhKuTAx+sWERzezdLnthAuJ87b9+2AFcnR2sX1UJrfVwzuR7uJjipOQghJryWjm5WrEkj0MuVx65MYX9FE8uf28rv391DRc/dyauWp9pUMACjOsW3hIMQYsL7w7q9FNa28NpN8/HzcOHEuABuPz2RVV/kAPCPS2cS3zMP0kQh4SCEmNDeSy/lrR0lrDg9gflxAZbtK05PYF/5QUJ8XLl8TqQVS2gdEg5CiAmrqLaV372zhxNi/FlxRmK/fU6ODjz3k0Gb4ycEGcoqhJiQuowmVqxJAwVPLEvpdyeykHAQQtgxrTX7K5owDXKj2GOfZ5Ne3MDfLplBpL+HFUpn2yQchBB2a833xZz9+Ab+8en+ftu/za3h39/kceWcKM6bGW6l0tk2CQchhF3KqWzigff34uniyDMb8ticWwNAXUsnd76ezqRAT/5wwTQrl9J2STgIIexO33mQPlq5iLhAT+56PZ3a5g5+/VYGDa1drFqWioeLjMkZioSDEMLuPPRxlnkepMtnERPgyZPLZ9PQ2sUFT21m/b4q7lmSxPQIX2sX06ZJOAgh7MoX+yp58dsDXH9yLIuTzPMgTQv34TdLkyhtaGPxlCBuODnWuoUcB6ROJYSwG5UH2/nVmxlMC/PhnnOS+u27bkEsEX7uzI8PGNVpJ+yF1WoOSqkopdRXSql9Sqm9SqmVPdsNSqnPlVI5PT/9rVVGIcT4YTRp7no93TIPkptz/3mQlFL8KDkUHzdnK5VwfLFms1I38Eut9VRgPvBzpdQ04F7gC611IvBFz2shhDisZzbk8W1eLX+8YBoJwRNrHqTRYLVw0FqXa6139jxvAvYBEcCFwEs9h70EXGSVAgohxo20onoe/Sybc2eEccWcKGsXxy7YRIe0UioWSAW+A0K01uVgDhBg0JU1lFI3K6W2K6W2V1dXj1lZhRC25WB7FyvWpBHi48ZfL54xofoT2traaGpqGpXPtnqHtFLKC1gL3Km1Pjjcv1it9bPAs2Be7Gf0SiiEsFVaa+5/dw+l9W28cctJ+HrYV3+CyWSiqamJuro66uvrLT97H+3t7cyYMYNLLrlkxM9t1XBQSjljDoZXtdZv92yuVEqFaa3LlVJhQJX1SiiEsGVv7yzlvfQyfnHWZObEGqxdnGPS3d1NQ0PDgACoq6ujoaEBo9FoOdbBwQE/Pz/8/f2JiIjAYDAQHj46039YLRyUuYrwH2Cf1vrRPrvWAdcCD/X8fM8KxRNC2LiCmhbuf28P8yYZ+PniBGsX57A6OzstX/i9j97XjY2N/Y51cXHB39+f4OBgpkyZgr+/PwaDAX9/f3x9fXFwGJveAGvWHE4GrgF2K6XSe7b9FnMovKGUuhEoAi63TvGEELaqs9vEitVpODs68PiVKTg6WL+foaOjo9+Xf98AOLRfwN3dHYPBQHR0tOXLvzcAPD09baLfxGrhoLXeBAz1J3DGWJZFCDG+PPLZfnaXNvJ/V59AuJ/7mJ23bwDU1tZavvxra2tpaWnpd6yXlxcGg4H4+HjLF39AQAD+/v64ubmNWZmPldU7pIUQ4mhsyK7m2Q35XHViNOdMDx3xz+/q6rJ84R/689AA8Pb2xmAwkJiYSEBAgKUGYDAYcHFxGfGyjSUJByHEqMutaibYx/W4706uae7gF29kMDnEi/vPO/bpto1GY7/f+vsGwMGDB/sd21sDSExMxGAw9AuB8R4AhyPhIIQYVXtKG7nkX98yNcybN29dgIvTsXWomkyaX72ZwcH2Ll756bwB02McSmvNwYMHLV/+fQOgvr4erX8YAe/u7k5AQACxsbGWAOgNAVdX12Mq73gn4SCEGDWtnd2sWJOGq7MDGSWNPPp5NvcuSTryGwfxwuYCvt5fzYMXJpMU6mPZ3t7ebvnyr6mp6RcEXV1dluOcnZ0xGAyEhYWRnJyMwWAgMDAQg8GAh4csE3ooCQchxKh5YF0mBTUtvPrTE3k/o5xnNuSxMCGQhYmBR/U5u4rr+dcn6Zwf60q8LuP993dbwqBvP4BSCj8/PwIDA5k0aZKlBhAQEIC3t7dNjAIaLyQchBCj4v2MMl7fXsztixNYEB9IapQ/3x+o46430vlk5SICvAY217S3t1NTU2OpAdTU1FBVXUNNbS0XumiohM8/Bw8PDwICAkhMTCQwMLBfM5Cj4+Gbm8TwSDgIIUZccV0rv317N6nRfqw8MxEAdxdHnlyeyoVPb+LeNd9xz2nh1NbWUl1dbQmC5uZmy2c4ODhgMBio6nRiX3cIyxZN48SpsQQEBEgz0BiQcBBCjKhuo4mVa9JQmHjgR1Hk7M+y1AZqamq42q0aXdrNq6+aj3dzcyMwMJCEhAQCAgIICgoiMDAQPz8/PtpTycOr07h9cQKX/WiKdS9sgpFwEEIcl66uLmpqaqiurqa6upotmQeIqalhlmMH77y6zXKcr68vgYGBzJtzAh/nNLOj0sgT15/C7LjQQfsCemsfs/vUPsTYkXAQQgxLR0dHvxDofTQ0NFiOUUrRYHTFy9efhbPiLbWAwMDAfvcEzFvUyTmPb+DX7+7n/TuC8HDp/1XUW/sAeGJZKs6ONrG6wIQi4SCE6Kejo4Pq6mqqqqqorq42dwpXVfW7OczR0ZHAwEAiIyNJSUkhKCgIF08/rnllL+4ezrz/84V4ug799WLwdOHxK1O46j/f8cC6TP5+2cx++5/4IoedRQ08uTyVKIP0L1iDhIMQE1RnZ6flt//eIDg0BJycnAgMDCQmJoagoCDLw9/fv9/soFprbn55BzWtXbxz3bzDBkOvBQmB/OzUeP71dR6nTA7i3JlhAGzJq+Wpr3K5/IRIzp81OtNRiyOTcBDCznV3d1NbW0tlZWW/EOjbHOTo6EhQUJAlBIKDgwkKCsLPz29YU0S/8l0Rn2dW8rtzpzI9wnfYZbvrrMl8m1fLvW/vYlaUL54uTtz1ejqTAjz54wXJx3K5YoRIOAhhJ7TW1NfXU1VVRWVlJdXV1VRWVlJbW2uZKsLBwYGAgAAiIiJISUkhODiY4ODgATWBo7G/ook/f5DJqZODuOHkSUf1XmdHB1YtS2Xpqo2sXJOOv4cLtS0dPH/tycOqfYjRI3/6QoxDra2tVFZWWmoDvWHQd7qI3gVjkpKSCAkJITg4mICAgBG9Say9y8gdq3fi7ebMI5fPwuEY1lWIDvDgLxdPZ+WadICjrn2I0SHhIIQNMxqN1NTUWIKg99H3ZjEPDw9CQkKYPXs2wcHBhISEmDuIx2DG0D9/mEl2ZTMv3TCPIO9jn6DuwpQIMssPUt/SedS1DzE6JByEsBHNzc1UVlZSUVFBVVUVFRUV1NTUYDKZgB/6BeLj4y01gZCQELy8vKxS3k/3VvDK1iJuWjSJUycHHffn/WbJ1BEolRgpEg5CjDGTyURtbS0VFRWWMKioqOg3gZy3tzehoaEkJiYSEhJCaGioTc0bVN7Yxj1rdzEjwpe7zz62WVaFbZNwEGIUdXZ29guA3mah7u5uwNxBHBwcTEJCgiUEQkJCjmvuoLzqZjxdnAj1Hd5SlAfbuyiqbR12O7/RpLlzTTqd3SZWLU895vUZhG2TcBBihLS2tlJRUUF5ebklDPqOFHJzcyM0NJQ5c+YQGhpKaGgogYGBI1obKKpt5cKnNuPr7sxHKxfh6374lde6jSau/+/37CyqZ/VN85kfF3DEc/zrq1y+K6jj4ctmMinQc6SKLmyMhIMQx6C5uZny8nLKysosgdDY2GjZ7+PjY1lUpjcIfH19R3U9gS6jiTvWpKGAyoPt/Pbt3Tz149TDnnPVl7nsKKzH38OZu15P56MVi/D3HLoje0dhHY9/kcMFs8K57ITIUbgKYSskHIQ4gqamJsrKyigvL7cEQt/RQgaDgaioKObOnUtYWBihoaFWmVL60c+zyShu4F9XzeZAbQv/+GQ/p2wP5Mq50YMe/11+LU99mcOlsyO5/uRYLv7XZu5Zu4tnrjlh0EBpbOtixep0wv3c+PPF02XhHDsn4SBEH83NzZSVlVnCoG8QKKUIDAwkdtIkwsLCiAgPJzQ01CbWGN6cW8P/fZPH8nlRLJ0Rhsmk2Zxbwx/XZXJCjIGE4P4jmhpaO7nz9XRiAjx54MJkvFyduOecJP784T5e/a6Iq+fH9Dtea8197+ym8mA7b956Ej5uh2+uEuOfhIOYsNra2ixBUFZWRmlpKU1NTZb9gYGBxMfHExYWRnh4OCEhIRhx5JJ/f4t3Fbw6L8omZgutbe7grtfTiQv05P7zpgHg4KB49IoUznl8A3esTuOd2xbg5mzu29Bac8/aXdQ0d/D2z07Gq+dO5BtOnsSGnBoe/CCTubEGpoR6W87x5vYSPthVzt1nTyE12n/sL1KMOQkHMSF0dXVRUVFBaWmpJQjq6uos+w0GAzExMYSHhxMeHk5YWNigN5H99p3d7Cs3T0z3+Ppsqw/j1Fpz91u7aGjt4sXr5/Wb+jrEx41HLp/FjS9t5++fZPGH881zFb22rYhP91by26VJzIj8YYSSg4Pin5fPYskTG7lj9U7W3b4QN2dH8qqb+cO6vSyID+DWU+PH/BqFdUg4CLujtaampobS0lLLo7Ky0nIzmY+PD+Hh4aSkpBAREUFYWBju7u5H/NxP9pTz2ndF3HJqHA0tXfzr6zxOTghkQXzgaF/SkF769gBfZlXxx/OnMS3cZ8D+M6aGcN2CWP67+QCLEgOJ9PfgT+9nsigxkJ8ujBtwfJC3K/+8YhbXvrCNv3y4j9+dN5U7XkvDzdmBx65MwfEYpscQ45OEgxj3WltbKS0tpaSkhJKSEkpLS+no6ADA1dWV8PBwFixYQEREBBEREXh7ex/hEwcqbWjj12/tYmakL788awrdJhPfF9Zx1+vpfLzyFAyHGeEzWjLLDvLXj7I4IymYaxfEDnncvUuS2Jpfy6/e3EWApwvebk7884qh50E6dXIQNy2axHMbC8ipaiKz/CDP/2QOIT7Du29C2AcJBzGumEwmqqqqLEFQUlJCbW0tYO4wDgkJYfr06URGRhIREUFgYOBxj6oxmjR3rUnHaNKsWma+6csFB55cnsrFT3/Lr9/axXM/6T/C59XvCvlmfzWrlqda2vpHUmtnN3es3omfhzP/uGzmYa/RzdmRp36cynlPbqKupZMXr59LsPfhv+jvPjuJrfl1bM2v47oFsZw5LWSkL0HYOAkHYdPa29spKSmhuLiY4uJiSktL6ezsBMwTzkVFRZGSkkJkZCTh4eGjMtncU1/msu1AHY9eMYvYPjd9JYf7cu+SJP70QSYvby3kJyfFApBWVM/v39uL0aR56OOsUVmX4MEPMsmvaeGVG08kwOvIo6USgr159po5VDd1cNqU4CMe7+LkwL+vns3bO0u5+ZSBzU/C/kk4CJuhtaaxsZGioiKKioooLi6mqqoK+KFWMHPmTKKiooiKisLPz2/Ux9p/f6COJ77I5uLUCC6ZPfCmr+tPjmVTbg1//nAfc2MNRPi7s2JNGqE+bixKDOTFbw+wMCFwRH/z/mh3Oau3FfOz0+I5OWH4/R2nHOXkeJH+Hqw4I/FoiyfshOq9tX88mzNnjt6+fbu1iyGOUm8TUW8YFBUVWYaSurq6EhkZaQmCiIiIMb+foLG1i6WrNuLooPhwxUK8hxjbX9vcwTlPbMTP3ZnJId58sreCN26Zz/QIXy56+lsqGtv45M5TRqTNvqS+laVPbGRSkBdv3XqSTQylFeOXUmqH1nrOYPuk5iDGjNFopLy8nMLCQgoLCykqKrJ0HPv4+BATE0N0dDRRUVEEBwcf88pkI0Frzb1v76LyYDtv/WzBkMEAEODlymNXpHDNC9+RU9XML8+azAkxBgCeXJ7K+U9u4q7X03n5xhOHNdqnqb2LZzfkc/X8mH6B0m00ceeadEwaVi1LkWAQo0rCQYya7u5uSktLLWFQXFxsWaksMDCQ5ORkSyCM9rxDR2vN98V8vKeCe85JIiXK74jHL0wM5LdLprKv4iC3LU6wbE8I9uKPF0zjnrW7eWZDHredlnCYT+m9E3kP6zLK+K6gjtU3zbcEypNf5rK9sJ7Hr0whJkAmvBOjy6rhoJR6ATgPqNJaT+/ZZgBeB2KBA8AVWut6a5VRDJ/RaKS0tJSCggJLGPROTR0SEkJqaioxMTHExMTg6Wm7X265VU088P5eFiYEcstRdMbeNMSxV8yJYkN2DY9+ls1JcQGHvcN47c5S1mWUcXJCAJtza3n6q1xWnJHItoI6nvwyh0tmR3BRasRRX5MQR8uqfQ5KqVOAZuB/fcLhH0Cd1vohpdS9gL/W+p7DfY70OViHyWSivLycgoICDhw4QFFRkaVmEBoaSmxsLLGxsURHRw/rJjNb0N5l5KKnN1PV1MEnKxcRPEJj+xvbulj6xEYcHOCjFYsGbabKr27mvCc3MTPSl1d/Op9fvpHOuowynvvJHO5/dw8uTg58sGKRZboLIY6XzfY5aK03KKViD9l8IXBaz/OXgK+Bw4aDGBtaa2pra8nPzyc/P58DBw5Y+gyCg4NJTU21BMJ4CYNDPfRxFlkVTfz3urkjFgwAvu7OrFqewhXPbOV37+7h8StT+jWjdXabWLkmHRenH+5EfvCi6ewsauDGl7bj5KB4+7YFEgxizNjiv7QQrXU5gNa6XCk16KBspdTNwM0A0dGDT0ksjl9LS4slDPLz8zl40DyvkJ+fH9OmTSMuLo7Y2FirrWM8kr7YV8mL3x7g+pNjWZx05HsBjtYJMQZWnpHIo59nc0piEJf2WQ/h4U+z2F3ayDPXnECYrzlYvd2cWbU8laue28ovfjSFmZF+I14mIYZii+EwLFrrZ4FnwdysZOXi2A2j0UhxcTG5ubnk5+dTXl4OmFcxi4uLszz8/e1rZs7Kg+3c/dYupoX5cO+S0ZtM7+eLE9iUW8Pv39vD7Bh/JgV68k12Nc9tLOCa+TGcnRza7/iUKD/Sfv8jWYpTjDlbDIdKpVRYT60hDKiydoHsXWNjIzk5OeTm5lJQUEBnZycODg5ERkayePFiy7TV1hxaOppMJs0v3kinrdPIquWpuDqN/HQXvRwdFI9fmcKSJzayYnUaz1xzAr98I50pId7cd+7UQd8jwSCswRbDYR1wLfBQz8/3rFsc+9NbO8jJySEnJ4fq6moAfH19mTFjBgkJCUyaNMkmFrEZitaa1k4jnoO0wbd0dB/V9mc25LM5t5a/XzpjwKI4oyHcz52/XzqTW1/ZwdJVG2nrNPLqT+ePyhxMQhwraw9lXY258zlQKVUC/AFzKLyhlLoRKAIut14J7UdbW5slDHJzc2lvb8fBwYGYmBhSUlJITEwckUnqxsrdb+3i88xK3r99IdEBPyzJ+fRXuTyxPodXfnoi8yYZLNs/2FXGnWvS+fulM/u19acXN/DPz/Zz7owwrpgTNWblP2d6KFedGM2r3xXx4EXT+y2sI4QtkOkz7Fh9fT1ZWVns37+foqIitNZ4enqSmJjI5MmTiYuLs+nawVDe3lnCL97IQCmYGelnmUZi+4E6rnx2KyatCfNx4+OVp+Dr4UxxnXnKiebObtydHfngjoXEBXnR1N7Fuas2YTRpPlq5CF/3sV36sstoYndpI6lRoz9HlBCDsdmhrGJkaa2prKxk3759ZGVlWSatCw4OZuHChUyePJmIiIhx/UV0oKaF+9/dw7xYA9ecFMMdq9N47PNsbjk1npVr0onwc+ehS2dw7QvbuPftXaxansrKNWkAvHnLSfz0f9tZsSaNt392Mve/u4eS+lbeuOWkMQ8GAGdHB2bLkpvCRkk4jHNaa0pLS8nMzCQrK4v6evPN5NHR0fzoRz9iypQpGAyGI3zK+NDZbWLFmjScHB14bFkKEX7ubM6t4d/f5PFtXi2VB9t589aTSI3251c/msLfPs7iyme2sLOogSeXpzIn1sA/Lp3JzS/vYNmz5u2/OGsyc2Lt489HiJEk4TAOaa0pKSkhMzOTzMxMDh48iIODA3FxcSxcuJApU6bY9PQUAB3dRm57ZSdzJxn6rUustea+d/eggD9fNL1fLeefn+9nV0kj/3f1bCL8zPcC/P78aXx/oI704gZ+fc4Uy9QUNy2KY1NuDRtzarj8hEjOnxUOwI+SQ7l6fjSvbC1i3iQDP198+LmOhJioJBzGCa01FRUV7Nmzh71799LY2IijoyPx8fGcfvrpTJkyBTe38bOM498/3s8XWVV8ub+KGRG+lnUJXvmuiNe+KwJgWrgPV50YA8DGnGqe+SafH58YzTnTwyyf4+HixPPXzuXLrCqu77NUpkPPkNE3d5RwzfyYfuf+3bnTiPT34JLUCFkTWYghSIe0jauvr2f37t3s3r2bmpoaSw0hOTmZpKSkcRUIvb7KquL6F79n+bwothXU0dTezccrF1HT3MkFT21iflwAJq35/kAd79++EH9PF5b0rJew7vaFuLvIkE8hRoJ0SI8z7e3t7N27l4yMDIqLiwFzH8K5557LtGnT8PDwOMIn2K6qg+386s0MkkK9+cP5yeRXt3DR05v51ZsZlDa04e3mzCOXz0KjWfL4Ru5YnUaIjxuNbV3874Z5EgxCjBEJBxuhtaagoIC0tDT27duH0WgkMDCQM844gxkzZuDr62vtIh43k0nzyzczaOnsZs1y801f08J9+M3SJB54PxOAl26YR5C3eXjtI1fM4vr/fk9WRRMPXJDM1DAfaxZfiAlFwsHKDh48SFpaGmlpaTQ2NuLm5sbs2bNJSUkhLCxsXA87PdRzG/PZmFPDXy+eQWLIDzd9XbcglgM1LUT6e3Bqn3WOF08J5rdLkyhraOcnJ8UM9pFCiFEi4WAFWmtyc3PZsWMH2dnZaK2Ji4vjzDPPJCkpCScn+/tryShu4OFP97NkeijL5/W/E1kpxQMXTh/0fTefEj/odiHE6LK/byEb1tbWRlpaGtu3b6e+vh5PT09OPvlkZs+ebXeznPbV3NHNijVpBHu78tAlM+2qNiSEvZJwGAM1NTV89913ZGRk0NXVRXR0NKeffjpTp07F0dH+O1h//+4eiutaWXPzSfh6jP2dyEKIoyfhMEq01hQVFbF582ZycnJwdHRkxowZnHjiiYSGhh75A+zEO2klvJ1WysozEvtNhCeEsG0SDiNMa012djabNm2ipKQEDw8PTj31VObOnWvzdy2PtMLaFn73zh7mxvpzx+lyJ7IQ44mEwwjRWrN//36++eYbKioq8PPzY8mSJaSmpuLsPPGaUjq7TaxYnWZe3GZZKk6OsmCNEOOJhMNx0lqTl5fHF198QUVFBQaDgQsvvJAZM2ZMiP6EoTz6eTYZJY38+6of5kESQowfhw0HZR5WEqm1Lh6j8owrJSUlrF+/nsLCQvz8/LjooouYMWOG3S6nOVybcmr4v2/yWD4viiUzwo78BiGEzTlsOGittVLqXeCEsSnO+HDw4EHWr1/P7t278fT0ZOnSpcyePXtC1xR61TZ3cNcb6SQEe/H785KtXRwhxDEaTrPSVqXUXK3196NeGhtnNBrZvHkzmzZtwmQysWjRIhYuXIiLi4u1i2YTtNbc/dYuGtu6eOl6mQdJiPFsOOGwGLhFKVUItAAKc6Vi5qiWzMYUFxfz/vvvU11dzdSpUznrrLPs+sa1Y/Hitwf4MquKP54/jWnhMg+SEOPZcMJhyaiXwoZ1dXWxfv16tm3bho+PD8uXL2fy5MnWLpbNySw7yN8+yuKMpGCu7bOughBifDpiOGitCwGUUsHA+Fs84DhUVlaydu1aqqurmTdvHqeffjqurq7WLpbNae3s5o7VO/HzcOYfl8n0GELYgyOGg1LqAuCfQDhQBcQA+wC77W3UWrN9+3Y+/fRT3NzcuPrqq4mPlwnghvLgB5nk17Twyo0nEuAl4SmEPRhOs9KDwHxgvdY6VSm1GFg+usWyHpPJxEcffcSOHTtISEjgoosumnB3Nh+Nj3aXs3pbMT87Ld6y1KcQYvwbzoD8Lq11LeCglHLQWn8FpIxusayjo6OD1157jR07drBw4UJ+/OMf23wwdBlN/O2jfewsqh+1c+wta+RP72fS3mXst72kvpV71+5iVpQfvzhL+mGEsCfDqTk0KKW8gI3Aq0qpKqB7dIs19lpbW3nppZeorq7m/PPPZ/bs2dYu0rA89nk2z2zI5+20Uj5ZuWjEm3UOtndxy8s7KKlvo8to4sGLzOsudBtN3LkmHZOGVctScJbpMYSwK8P5H70B8ANWAp8AecD5o1imMdfZ2clrr71GXV0dV1111bgJhm9za/j3N3ksnhJEY1sXv3ozA631iH2+1pr73tlDeWM7Z04N4eWthXy2twKAVV/msr2wnj9fNJ2YANuuXQkhjt5wwkEBnwJfA17A6z3NTHbBZDKxdu1aysrKuPTSS8dNx3NdSyd3vp7OpEBPnr5qNr87dypf7a/mv5sPjNg53tpRwvsZZfzirMk8fVUq0yN8+PXaXbyXXspTX+ZwyewILkqNGLHzCSFsxxHDQWv9gNY6Gfg55hFL3yil1o96ycaA1poPPviA7Oxsli5dSlJSkrWLNCxaa379VgYNrV08uTwVDxcnrpkfw5lTQ3jo4yz2lDYOeE9aUT1ZFQcHbM+vbmZbQd2g2/+wbi8nxQVw66nxuDo5smpZKp3dJlauSSfa4MGfhljaUwgx/h1NQ3EVUAHUAsGjU5yxlZ+fT1paGosWLWLOnDnWLs6w/W9LIev3VXHvkiSSw30B8zrM/7hsJv6ezqxYk0Zr5w/dQjmVTSx/bivLnt1KeWObZXttcwfLnt3Kj5/bSkZxg2V7R7eRO1an4erkwGNXpuDoYL5vIS7Ii79ePINAL1dWLU/Fy1Um9RXCXh0xHJRSP1NKfQ18AQQCN9nL1Bnx8fFcffXVLF682NpFGbZ95Qf5y0f7OD0pmOtPju23z+DpwmNXplBQ08ID6zIBaO8yf9F7uDjR2W3uRDaaNFpr7lm7i4bWLgyeLqxYk0ZzhzlQHv5kP3vLDvLwZbMI9e1/3+NFqRF8f98ZzIz0G4vLFUJYyXB+9YsB7tRap49yWaxivPQxALR1mr/ofd2deXiIO5EXxAdy22nxPP1VHosmB7L9QD1ZFU3897q51LZ08qs3M/j317l4uzmzfl8Vfzh/Gsnhvix7dgu/f28PF8wK5/lNBVx7UgxnTgsZtBxyB7QQ9m8402fcOxYFsZYN2dUsTAjEwWF4X3j1LZ14uTkNGLrZ3NGNAjwPaWrp7DbR2tmNn0f/mVtNJk1tSydB3gOHnlY1tRPsPXCmkj99kEledTMv33D4O5HvPHMym3NrufvNXbR1Gblx4SQWJwWjtWZDdjWPrc/BUSlOTwrmugWxKKVYcUYij6/P4ZM9FSSFevObpVOH9echhLBPE3pw+sacan7ywjZe2FwwrOOL61o59eGvuO6/2zCafhgy2tTexXmrNnL+U5to6fihrd9o0lzzn+847ZGvKWto6/dZ9769iwUPfUF6n7Z+gH9/nce8v3zB+xll/bZ/vLuc1duKuPmUOBYmHv5OZGdHB55cnoqTgyI53IdfnzMFMP/G/+eLpxPu54afR//ax+2LE5gXa8Bo0jy5PBU3Z5luW4iJTI3kuHhrmTNnjt6+fftRv09rzS0v7+Cr/VW8c9vJTI/wHfLYbqOJK57Zwu7SRrqMml+fM4XbTktAa82dr6fzfkYZGrhsdiQPXz4LgCfW5/DY+mycHRWp0f6svmk+jg6K99JLWbkmHWdHRZivOx+uWIi3mzNpRfVc9n9bcFDg5uTIRysXEWXwoLShjSWPb2BSoCdv/WzBsG84K2tow8fdeUDHcUNrJ11GPaDW0tZppKa5gyiDx9H9QQohxiWl1A6t9aCjcWy25qCUOkcptV8plauUGpWmLaUUf790JgGertyxOq3fb/2HeuKLHHYWNfDoFSmcOyOMRz/LJr24gbd3lvJeehl3njmZn5+WwJs99wZ8f6COJ77I5uLUCB66ZCbbCup4+qtciuta+d07e5gd7cf/bjiRkvpW7n93D03tXaxYk0aojxvv3HYyACvXpNHRbeSu3juRl6ce1Z3I4X7ug44o8vNwGbQ5y93FUYJBCAHYaM1BKeUIZANnASXA98ByrXXmYMcfa82h19b8WpY/t7Xfb/19bcmr5cfP/7C/sa2LpU9sBMy/hSdH+LL6pvmYtObKZ7aQU9mMt5sTTo4OfLhiIV6uTtz1ejrv7yonNsCDqqYOPlphrhX01i6mhHiTW93MG7fM54QYA+syylixOo0pId7sr2zisStncXFq5DFfoxBCHGo81hzmAbla63ytdSewBrhwtE42Py6AOxabf+t/L7203776lk7uej2dSQGe/PEC8yzlvu7OrFqeQsXBdpwcHXi8514AZ0cHnliWCkBVUwerlqfi7eaMUooHL5pOhJ87edUt/PXiGZbf0G8/PYF5kwzsr2zizjMSOSHGAMAFs8K5/IRI9lc2cUlqhASDEGJM2WrN4TLgHK31T3teXwOcqLW+vc8xNwM3A0RHR59QWFh4XOfsNpq48tmtZFc0Wdr6tdbc/PIOvh6iT+Kr/VUEeroyI7L/9vTiBhrbujh1clC/7QU1LewubeSCWeH9ttc0d7A+s5LL50RZbjgD8yI676aVcWFK+IBRUEIIcbwOV3Ow1XC4HDj7kHCYp7W+Y7Djj7dZqVdxXStLV20kIdiLN245iTXbirj/vb387typ/HRR3HF/vhBC2JLx2KxUAkT1eR0JlA1x7IiJMnjwt0tmkFbUwN1vZvDnD/dx6uQgbjh50mifWgghbIqthsP3QKJSapJSygVYBqwbixOfNzOcK+dE8W56Gd5uzjxy+axh3yAnhBD2wiYbsrXW3Uqp2zFPFe4IvKC13jtW5//DBdMwas0Vc6IGHfIphBD2zibDAUBr/RHwkTXO7eHixCODDGkVQoiJwlablYQQQljRhA6HgwcP8sorr9DQ0GDtogghhE2Z0OHQ3t5OaWkpr7zyCq2trdYujhBC2IwJHQ7BwcEsW7aMhoYGVq9eTVdXl7WLJIQQNmFChwNATEwMl1xyCSUlJaxduxaTyWTtIgkhhNVN+HAAmDZtGkuWLGH//v288cYbdHZ2WrtIQghhVRIOPebNm8eSJUvIzs7mxRdfpKmpydpFEkIIq5Fw6GPevHksW7aMmpoann/+ecrLy61dJCGEsAoJh0NMnjyZ66+/Hq01zz//PFu2bMEWJycUQojRJOEwiLCwMG699VYSExP57LPPeO2112hubrZ2sYQQYsxIOAzBw8ODK6+8kqVLl1JQUMDTTz9NWlqa1CKEEBOChMNhKKWYO3cut956K8HBwaxbt46XX36Z2tpaaxdNCCFGlU0u9nO0Rmqxn8PRWrNjxw7Wr19PV1cXJ554Iqeccgpubm6jel4hhBgth1vsx2ZnZbU1SinmzJlDUlISX3zxBVu2bGHXrl2cdtpppKam4ujoaO0iCiHEiJGawzEqKyvj008/paioCIPBwOLFi0lOTkYpWRhICDE+jLs1pI+WNcIBzE1N2dnZfPnll1RVVRESEsKpp55KUlKShIQQwuZJOIwyk8nEnj17+Oabb6irqyM4OJhFixYxbdo0HBykz18IYZskHMZIb0hs2LCB2tpa/P39WbBgAbNmzcLZ2dnaxRNCiH4kHMaYyWRi//79bN68mdLSUjw8PJgzZw5z587Fy8vL2sUTQghAwsFqtNYUFhayZcsWsrOzcXR0ZPr06cydO5eIiAhrF08IMcHJUFYrUUoRGxtLbGwstbW1fPfdd2RkZJCRkUF4eDhz584lOTlZmpyEEDZHag5jrKOjg4yMDLZv3051dTWurq7MnDmTE044gZCQEGsXTwgxgUizkg3SWlNUVMTOnTvZu3cvRqOR8PBwUlJSmDFjhtx5LYQYdRIONq6trY2MjAzS09OprKzEycmJKVOmMGvWLOLj42U4rBBiVEg4jBNaayoqKkhLS2PPnj20tbXh6enJ9OnTmTFjBuHh4XJznRBixEg4jENGo5GcnBx27dpFdnY2RqORgIAAkpOTmT59OkFBQdYuohBinJNwGOfa29vJzMxk9+7dHDhwAIDg4GCSk5NJTk4mICDAugUUQoxLEg52pKmpiczMTPbu3UtxcTEAISEhTJ06lWnTpkmNQggxbBIOdurgwYNkZmaSmZlpCYrAwECSkpJISkqSPgohxGFJOEwATU1N7Nu3j3379lFYWIjWGh8fHyZPnkxSUhIxMTE4Ock9j0KIH0g4TDCtra3k5OSQlZVFXl4eXV1duLi4kJCQwOTJk0lISMDT09PaxRRCWJlMnzHBeHh4MGvWLGbNmkVXVxcFBQVkZWWRk5NDZmYmAJGRkSQmJjJ58mRCQkKk+UkI0Y/UHCYQrTXl5eVkZ2eTk5NDWVkZAN7e3sTHx5OYmEhcXJzcnS3EBGFzzUpKqcuBPwJTgXla6+199v0GuBEwAiu01p8e6fMkHI5Nc3Mzubm55OTkkJeXR0dHB0opoqKiiI+PJz4+nrCwMLlDWwg7ZYvhMBUwAc8Av+oNB6XUNGA1MA8IB9YDk7XWxsN9noTD8TOZTJSUlJCbm0tubi7l5eUAuLu7ExcXZ3n4+flZt6BCiBFjc30OWut9wGDt3BcCa7TWHUCBUioXc1BsGdsSTjwODg5ER0cTHR3N6aefTktLC3l5eeTn55Ofn8/evXsBMBgMlqCIjY3F3d3dyiUXQowGW+uQjgC29nld0rNtAKXUzcDNANHR0aNfsgnG09OTmTNnMnPmTLTWVFdXW4Kid8pxgLCwMCZNmsSkSZOIjo7GxcXFyiUXQoyEUQsHpdR6IHSQXfdprd8b6m2DbBu03Utr/SzwLJiblY6pkGJYlFIEBwcTHBzM/PnzMRqNlJaWkp+fT0FBAVu3buXbb7/FwcGBiIgIywJHUVFRspCREOPUqIWD1vrMY3hbCRDV53UkUDYyJRIjxdHR0dIEddppp9HZ2UlxcTEFBQUcOHCATZs2sXHjRhwdHYmIiCAmJobY2FgiIyOlZiHEOGFrzUrrgNeUUo9i7pBOBLZZt0jiSFxcXCyjm8C82l1hYSGFhYX9wsLBwYHw8HCio6OJiYkhOjpahs0KYaOsNVrpYuBJIAhoANK11mf37LsPuAHoBu7UWn98pM+T0Uq2raOjg+LiYg4cOEBhYSFlZWWYTCbAPGlgby0kJiYGb29vK5dWiInD5oayjjQJh/Glq6uLkpISioqKKCoqori4mK6uLgD8/PyIjo4mKiqK6OhogoKC5O5tIUaJzQ1lFRObs7OzZYQTmBc2qqiooLi4mKKiIvLy8ti1axcArq6uREVFWR4RERHSbyHEGJBwEFbX23EdERHB/Pnz0VpTX19vqVUUFxeTm5sLmEdOhYSEEBkZSVRUFJGRkfj7+0vtQogRJs1KYlxoa2ujpKSk36OzsxMwTzQYGRlJRESE5aerq6uVSyyE7ZNmJTHuubu7k5iYSGJiImCe7qO6urpfWGRnZ1uODwoKstRGIiMjCQ4OljmihDgKUnMQdqO9vZ3S0lJKSkosP9va2gBwcnIiPDyc8PBwS2j4+flJc5SY0GS0kpiQevsuSktLLY/y8nKMRvM8ju7u7v0CIzw8XIbSiglFmpXEhKSUwmAwYDAYmDFjBmAeGVVVVUVpaSllZWWUlpayadMmen9J8vb2tgRGWFgY4eHhsmqemJAkHMSE4ujoSFhYGGFhYZZtnZ2dVFRUUFZWZnns37/fst/Hx8cSFhIYYqKQcBATnouLi+Uu7V7t7e2WwCgvL6esrIysrCzLfh8fH8LCwggNDbWEhre3t/RhCLsh4SDEINzc3Cyzy/bqGxgVFRWUl5f3q2F4enoSGhpqCYzQ0FAMBoMEhhiXJByEGKbBAqO3Sao3LMrLy9myZYtl7igXFxdLYPQ+goKCcHKS/3rCtsm/UCGOw2BNUt3d3VRXV1NeXm4JjrS0NMv8UQ4ODgQFBRESEkJoaKjlp4eHh7UuQ4gBJByEGGFOTk4DOr211tTV1VnCorKykoKCAsscUmAeKRUSEtIvNAICAuTmPWEVEg5CjAGlFAEBAQQEBJCcnGzZ3tLSQmVlpSUwKioqyM/PtzRLOTk5WWoZfR9SyxCjTcJBCCvy9PQkLi6OuLg4y7bu7m5qamqorKy0PLKzs0lPT7cc4+XlRUhICMHBwZaf0pchRpL8SxLCxjg5OVk6r/tqbm62hEVVVRWVlZVs27bNcsd3b+2kd73v3uCQWWvFsZBwEGKc8PLywsvLy7IcK5gnIKytrbWERVVVFeXl5WRmZlqOcXZ2JigoiKCgoH7BIfdliMORcBBiHOsd+RQUFNSvL6Ozs5Pq6mqqqqosj7y8PDIyMizHuLq6Wpqj+gaHp6enhIaQcBDCHrm4uFhmn+2rtbXVEhqVlZVUV1ezb98+du7caTnG3d3dEhi9oREUFCShMcFIOAgxgXh4eBATE0NMTIxlm9aalpYWqqqqLMFRXV3N3r17aW9vtxzXGxqBgYH9wkOap+yThIMQE5xSytKf0XfUlNaa5ubmfoFRU1NDZmZmv9BwdXXtFxq9P2W9jPFNwkEIMSilFN7e3nh7ew8IjZaWFmpqaizBUVNTQ05OTr/htk5OTgQEBFgCozc0DAaDDLkdB+RvSAhxVPrWNPrOMwXmtb57axi9P0tKStizZ0+/9/v7+1sCo+/D3d19jK9GDEXCQQgxYtzd3QfMNQXQ1dVFbW2tJTB6H3l5eZb7NMDcJ9I3LAICAggMDMTPz0+mERljEg5CiFHn7Ow86I19JpOJhoaGfoFRW1tLVlYWra2tluMcHR0xGAyWsOj7U2obo0PCQQhhNQ4ODpalXCdPntxvX2trK7W1tf1Co7q6muzsbMvcU/BDbaN37qreh8FgwNHRcawvyW5IOAghbJKHhwceHh5ERUX12240Gqmvr7cER21tLbW1tWRnZ9PS0mI5TimFn5/fgNAICAjAx8dHRlIdgYSDEGJccXR0tPRJTJkypd++9vb2fqFRV1dHTU0NhYWFlvU04IeRVL01jL7PPTw8JDiQcBBC2BE3N7dB7wzXWtPU1GSpZfQ+KisrycrK6tdM5ebmZgmM3iav3ucTqX9DwkEIYfeUUvj4+ODj48OkSZP67evtFO8bGnV1dRQVFbF79+5+x7q7u/cLjb7h4ebmNpaXNOokHIQQE1rfTvHExMR++7q7uy39G3V1ddTW1lJfX8+BAwf6reIH/YPD39+/X3CMxxqHhIMQQgyhdyW+oKCgAfu6urqor6+nrq6u32Ow4Ohtqjo0OAwGg81OaCjhIIQQx8DZ2dkyzfmh+tY4+gZIaWkpe/fuRWvd73P6hkbf8PDx8bHazX9WCQel1MPA+UAnkAdcr7Vu6Nn3G+BGwAis0Fp/ao0yCiHEsTpcjcNoNNLQ0EBdXV2/AOmdn6rvHeMODg74+flZQqNvgPj7++Ps7Dx61zBqn3x4nwO/0Vp3K6X+DvwGuEcpNQ1YBiQD4cB6pdRkrbXxMJ8lhBDjhqOjo2Xo7KFMJhNNTU2W4Oj7s7i4mI6Ojn7He3l5MX36dM4+++wRL6dVwkFr/Vmfl1uBy3qeXwis0Vp3AAVKqVxgHrBljIsohBBjzsHBAV9fX3x9fQeMqtJa09bW1i806uvr8fX1HZWy2EKfww3A6z3PIzCHRa+Snm1CCDGhKaUsd40feh/HaBi1cFBKrQdCB9l1n9b6vZ5j7gO6gVd73zbI8XqQbSilbgZuBgbMACmEEOL4jFo4aK3PPNx+pdS1wHnAGfqHrvsSoO9EKpFA2RCf/yzwLMCcOXMGDRAhhBDHxipjpJRS5wD3ABdorVv77FoHLFNKuSqlJgGJwDZrlFEIISYya/U5PAW4Ap/33PyxVWt9q9Z6r1LqDSATc3PTz2WkkhBCjD1rjVZKOMy+vwB/GcPiCCGEOISsuyeEEGIACQchhBADSDgIIYQYQPWdAGq8UkpVA4XH8RGBQM0IFWe8mIjXDBPzuuWaJ46jve4YrfXACaCwk3A4Xkqp7VrrOdYux1iaiNcME/O65ZonjpG8bmlWEkIIMYCEgxBCiAEkHMyetXYBrGAiXjNMzOuWa544Ruy6pc9BCCHEAFJzEEIIMYCEgxBCiAEmdDgopc5RSu1XSuUqpe61dnlGg1IqSin1lVJqn1Jqr1JqZc92g1Lqc6VUTs9Pf2uXdTQopRyVUmlKqQ96Xtv1dSul/JRSbymlsnr+zk+y92sGUErd1fPve49SarVSys0er1sp9YJSqkoptafPtiGvUyn1m57vt/1KqaNaS3TChoNSyhF4GlgCTAOW96xhbW+6gV9qracC84Gf91znvcAXWutE4Iue1/ZoJbCvz2t7v+4ngE+01knALMzXbtfXrJSKAFYAc7TW0wFHzGvR2+N1vwicc8i2Qa+z5//5MiC55z3/6vneG5YJGw6Y16bO1Vrna607gTWY17C2K1rrcq31zp7nTZi/LCIwX+tLPYe9BFxklQKOIqVUJHAu8HyfzXZ73UopH+AU4D8AWutOrXUDdnzNfTgB7kopJ8AD8yJhdnfdWusNQN0hm4e6zguBNVrrDq11AZCL+XtvWCZyOEQAxX1e2/161UqpWCAV+A4I0VqXgzlAgGArFm20PA78GjD12WbP1x0HVAP/7WlKe14p5Yl9XzNa61LgEaAIKAcatdafYefX3cdQ13lc33ETORyGvV61PVBKeQFrgTu11getXZ7RppQ6D6jSWu+wdlnGkBMwG/i31joVaME+mlIOq6eN/UJgEhAOeCqlrrZuqWzCcX3HTeRwGPZ61eOdUsoZczC8qrV+u2dzpVIqrGd/GFBlrfKNkpOBC5RSBzA3GZ6ulHoF+77uEqBEa/1dz+u3MIeFPV8zwJlAgda6WmvdBbwNLMD+r7vXUNd5XN9xEzkcvgcSlVKTlFIumDtu1lm5TCNOmddh/Q+wT2v9aJ9d64Bre55fC7w31mUbTVrr32itI7XWsZj/br/UWl+NHV+31roCKFZKTenZdAbmJXft9pp7FAHzlVIePf/ez8Dct2bv191rqOtcByxTSrkqpSYBicC2YX+q1nrCPoClQDaQB9xn7fKM0jUuxFyV3AWk9zyWAgGYRzbk9Pw0WLuso/hncBrwQc9zu75uIAXY3vP3/S7gb+/X3HPdDwBZwB7gZcxr1NvddQOrMferdGGuGdx4uOsE7uv5ftsPLDmac8n0GUIIIQaYyM1KQgghhiDhIIQQYgAJByGEEANIOAghhBhAwkEIIcQAEg5CHIOe2U9v63kerpR6y9plEmIkyVBWIY5BzzxVH2jzLKBC2B0naxdAiHHqISBeKZWO+eajqVrr6Uqp6zDPiukITAf+CbgA1wAdwFKtdZ1SKh7zlPFBQCtwk9Y6a6wvQoihSLOSEMfmXiBPa50C3H3IvunAjzFPj/wXoFWbJ8LbAvyk55hngTu01icAvwL+NRaFFmK4pOYgxMj7SpvXzmhSSjUC7/ds3w3M7JkhdwHwpnkqIMA83YMQNkPCQYiR19HnuanPaxPm/3MOQENPrUMImyTNSkIcmybA+1jeqM3raRQopS4H88y5SqlZI1k4IY6XhIMQx0BrXQts7lno/eFj+IirgBuVUhnAXuxwiVoxvslQViGEEANIzUEIIcQAEg5CCCEGkHAQQggxgISDEEKIASQchBBCDCDhIIQQYgAJByGEEAP8PzIgaIBXQT9BAAAAAElFTkSuQmCC\n", "text/plain": [ "
    " ] @@ -1459,425 +1498,6 @@ "(-1*confidence).plot(c='gray')" ] }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
    <xarray.DataArray 'var' (time: 100)>\n",
    -       "array([False, False, False, False, False, False, False, False, False,\n",
    -       "       False, False, False, False, False, False, False, False, False,\n",
    -       "       False, False, False, False, False, False, False, False, False,\n",
    -       "       False, False, False, False, False, False, False, False, False,\n",
    -       "       False, False, False, False, False, False, False, False, False,\n",
    -       "       False, False, False, False, False, False, False, False, False,\n",
    -       "       False, False, False, False, False, False, False, False, False,\n",
    -       "       False, False, False, False, False, False, False,  True,  True,\n",
    -       "        True, False, False, False, False,  True,  True,  True,  True,\n",
    -       "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
    -       "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
    -       "        True])\n",
    -       "Coordinates:\n",
    -       "  * time     (time) int64 0 1 2 3 4 5 6 7 8 9 ... 90 91 92 93 94 95 96 97 98 99\n",
    -       "    alpha    float64 0.05
    " - ], - "text/plain": [ - "\n", - "array([False, False, False, False, False, False, False, False, False,\n", - " False, False, False, False, False, False, False, False, False,\n", - " False, False, False, False, False, False, False, False, False,\n", - " False, False, False, False, False, False, False, False, False,\n", - " False, False, False, False, False, False, False, False, False,\n", - " False, False, False, False, False, False, False, False, False,\n", - " False, False, False, False, False, False, False, False, False,\n", - " False, False, False, False, False, False, False, True, True,\n", - " True, False, False, False, False, True, True, True, True,\n", - " True, True, True, True, True, True, True, True, True,\n", - " True, True, True, True, True, True, True, True, True,\n", - " True])\n", - "Coordinates:\n", - " * time (time) int64 0 1 2 3 4 5 6 7 8 9 ... 90 91 92 93 94 95 96 97 98 99\n", - " alpha float64 0.05" - ] - }, - "execution_count": 39, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "significantly_different" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -2036,8 +1656,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "59.1 ms ± 2.65 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", - "1.45 ms ± 44.8 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n" + "65.1 ms ± 1.78 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", + "1.44 ms ± 41.4 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n" ] } ], @@ -2066,7 +1686,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 47, @@ -2075,7 +1695,7 @@ }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAlVUlEQVR4nO3deXxU5dn/8c8VtoBsCqgI1KBlEUgMyL5GRcGC4AIKdYFqRVwrtiiiP8EFpYobLuXBpxZ8FLdWK0qrgBoRAZElIgIKSBQEAdEAQbAErt8fc0gnIftCksP3/Xrlxcw59zn3dZLhO2fuOXOPuTsiIhIuMWVdgIiIlDyFu4hICCncRURCSOEuIhJCCncRkRBSuIuIhJDCXY4oM/u3mQ07Av3EmZmbWeWS7tfMepjZl1H3U82sd0nsO9jfF2aWVFL7k6OT6Tr3cDOzIcAooA2wB9gATAf+4iH+45tZHJFjreLuGYXYzoFm7r6uENukAr9397lFqHMasMnd7yrstiJ50Zl7iJnZH4EngIeBE4ETgJFAN6BqGZZWYR16JSBS7rm7fkL4A9QhcqZ+cT7t+gHLgV3ARmB81LokImeV0e1Tgd7B7Y7AkmDbrcCjwfJY4AVgB5AGfAqcEKxLJnKWC3Aq8H7Q7gfgRaButr7+BKwAdgKvALG5HEclYFKwn6+BGwAHKufQ76+BD4N9/gC8EiyfF2yzB0gHLj30OwBuB74H/i/77yWo8w5gFfAT8LdDdQLDgfnZavWghhHAfuA/QX9v5fA7rgY8DmwOfh4HqkX/fYA/AtuALcDv8vhbJwP3AwsO9QfUC37vu4K/U1xU+5bAHOBH4EvgkgI+buKCYxwGfBv8ju8s6/8TR9uPztzDqwuRYHgzn3Z7gCuBukT+w15nZhcUsI8ngCfcvTaRoH41WD6MyJNLEyLhMRLYm8P2BjwInAScFrQfn63NJUBfoCmQQCQsc3IN0B9oC7QHBuVR933AbOBYoDHwJIC79wzWn+7uNd39leD+icBxwMlEAjknlwF9iPwemgP5DrO4+1QiwfpQ0N/5OTS7E+gMJAKnE3lCjd73iUR+142Aq4GnzezYPLodAlwRtD8VWEjkyeg4YDUwDsDMjiES7DOA44GhwDNm1jrYT0EeN92BFsDZwN1mdloedUkJU7iHV33gB48abzazBWaWZmZ7zawngLsnu/vn7n7Q3VcALwG9CtjHfuDXZlbf3dPdfVHU8nrAr939gLsvdfdd2Td293XuPsfdf3H37cCjOfQ92d03u/uPRM40E3Op5RLgcXffGLR9MJ+6TwZOcvd97j4/n+M8CIwL6szpSQrgqai+JxAJw5JwGXCvu28Lfkf3EAnnQ/YH6/e7+7+InJG3yGN/f3P39e6+E/g3sN7d5waPk9eIPDlC5Iky1d3/5u4Z7r4M+AfBk2YBHzf3uPted/8M+IzIk5McIQr38NoB1I8eI3b3ru5eN1gXA2BmnczsAzPbbmY7iZxl1y9gH1cTOUtdY2afmln/YPn/Ae8CL5vZZjN7yMyqZN/YzI43s5fN7Dsz20VkKCd7399H3f4ZqJlLLScRGR445Js86r6NyKuGxcGVKVfl0RZgu7vvy6dN9r5Pyqd9QZ1E1mPJvu8dnvUN47x+RxAZPjtkbw73D217MtApOBlIM7M0Ik80J0KBHzcF/dtJKVC4h9dC4BdgYD7tZgAzgSbuXgeYQiT4IPLSu8ahhmZWCWhw6L67r3X3oURetv8Z+LuZHROcRd7j7q2ArkTOAq/Moe8HiYzNJgRDO5dH9V1YW4gM6xzyq9wauvv37n6Nu58EXEtkuOHXeey7IFcVZe97c3A7++/wxELuezORoM1p36VpI/Chu9eN+qnp7tcF6/N63Eg5oHAPKXdPI/IS/hkzG2RmNc0sxswSgWOimtYCfnT3fWbWEfht1LqvgFgz6xeced9FZBwfADO73MwauPtBIm+cAhwwszPNLD54MthFZOjgQA5l1iIyjJBmZo2A0cU45FeBm82scTDmPCa3hmY22MwaB3d/IhKwh+rbCpxShP5vCPo+DhhL5M1fiAxHtDazRDOL5fD3FPLr7yXgLjNrYGb1gbuJvMIpbW8Dzc3sCjOrEvx0iBo3z+txI+WAwj3E3P0h4FYiwxDbiATJ/xC58mNB0Ox64F4z200kOF6N2n5nsP5/ge+InIVuiuqiL/CFmaUTeXN1SDB8cSLwdyLBvprIlSk5BdI9QDsiV63MAl4vxuE+S2Qo6DNgWT776gB8EtQ9E/iDu28I1o0HpgdDEZcUov8ZRN6k/Tr4uR/A3b8C7gXmAmuB7OP7fwVaBf39M4f93k/kiqQVwOfBsd1fiLqKxN13A+cSeQN2M5Ehlj/z3yf3XB83Uj7oQ0wiIiGkM3cRkRBSuIuIhJDCXUQkhBTuIiIhVC4mQapfv77HxcWVdRlylPvyy8gsvi1a5PUBT5HyY+nSpT+4e4Oc1pWLcI+Li2PJkiVlXYYc5ZKSkgBITk4u0zpECsrMcv0ktoZlRERCSOEuIhJCCncRkRAqF2PuIqVh//79bNq0iX378pvQMWLcuHEArF69ujTLEim02NhYGjduTJUqh02umiuFu4TWpk2bqFWrFnFxcZjlP2FhTEzkhayulpHyxN3ZsWMHmzZtomnTpgXeTsMyElr79u2jXr16BQp2kfLKzKhXr16BX4EeonCXUFOwSxgU5XGscBcRCSGFu8gRMn78eCZNmgTA3Xffzdy5c3Nt+89//pNVq1blun7KlCk8//zzQOTDV4X5EGBaWhrPPPNM5v3NmzczaFBe3ycuFZHCXQRg83L4ZXfkZ/PyUu/u3nvvpXfv3rmuzyvcMzIyGDlyJFdemdM3F+Yve7ifdNJJ/P3vfy/SvqT8UriLlKIJEybQokULevfunTl3DcDw4cMzA3XMmDG0atWKhIQE/vSnP7FgwQJmzpzJ6NGjSUxMZP369SQlJTF27Fh69erFE088keVVAMALL7xA165dadOmDYsXLwY4rE2bNm1ITU1lzJgxrF+/nsTEREaPHk1qaipt2rQBIm9C/+53vyM+Pp62bdvywQcfADBt2jQuuugi+vbtS7NmzbjttttyPN64uDjGjh1Lly5daN++PcuWLaNPnz6ceuqpTJkyJbPdww8/TIcOHUhISMi8BBXgggsu4IwzzqB169ZMnTo1c3nNmjW58847Of300+ncuTNbt25F8pbvpZBm1gR4nshXpx0Eprr7E2Y2HrgG2B40Hevu/wq2uQO4msj3Ut7s7u+WQu0iBXbLLbeQkpKSe4P/pPPz/si3ktWoYlC1Zr77TExM5PHHH891/dKlS3n55ZdZvnw5GRkZtGvXjjPOOCNLmx9//JE33niDNWvWYGakpaVRt25dBgwYQP/+/bMMl6SlpfHhhx8CkeCOtmfPHhYsWMC8efO46qqrWLlyZa51TZw4kZUrV2b+PlJTUzPXPf300wB8/vnnrFmzhnPPPZevvvoKgJSUFJYvX061atVo0aIFN910E02aNMm+e5o0acLChQsZNWoUw4cP5+OPP2bfvn20bt2akSNHMnv2bNauXcvixYtxdwYMGMC8efPo2bMnzz33HMcddxx79+6lQ4cOXHzxxdSrV489e/bQuXNnJkyYwG233cazzz7LXXfdlesxSsGuc88A/ujuy8ysFrDUzOYE6x5z90nRjc2sFZHvXWwNnATMNbPm7p7TFySLhNZHH33EhRdeSI0aNQAYMGDAYW1q165NbGwsv//97+nXrx/9+/fPdX+XXnppruuGDh0KQM+ePdm1axdpaWlFqnn+/PncdNNNALRs2ZKTTz45M9zPPvts6tSpA0CrVq345ptvcgz3Q8cZHx9Peno6tWrVolatWsTGxpKWlsbs2bOZPXs2bdu2BSA9PZ21a9fSs2dPJk+ezBtvvAHAxo0bWbt2LfXq1aNq1aqZv5szzjiDOXPmHNavZJVvuLv7FmBLcHu3ma0GGuWxyUDgZXf/BdhgZuuAjsDCEqhXpEjyOsMGYPNyvvwhcv7Ron4lOKltifSb3yVslStXZvHixbz33nu8/PLLPPXUU7z//vs5tj3mmGMK3I+ZUblyZQ4ePJi5rCDXSef1ncrVqlXLvF2pUiUyMjLybBcTE5Nlm5iYGDIyMnB37rjjDq699tos2yUnJzN37lwWLlxIjRo1SEpKyqy5SpUqmceYV9/yX4UaczezOKAt8Emw6EYzW2Fmz5nZscGyRsDGqM02kcOTgZmNMLMlZrZk+/bt2VeLVHg9e/bkjTfeYO/evezevZu33nrrsDbp6ens3LmT3/zmNzz++OOZQyW1atVi9+7dBe7rlVdeASJn3nXq1KFOnTrExcWxbNkyAJYtW8aGDRvy3XfPnj158cUXAfjqq6/49ttvS/wTu3369OG5554jPT0dgO+++45t27axc+dOjj32WGrUqMGaNWtYtGhRifZ7tCnw9ANmVhP4B3CLu+8ys78A9wEe/PsIcBWQ06nKYacD7j4VmArQvn373E8XRCqodu3acemll5KYmMjJJ59Mjx49Dmuze/duBg4cyL59+3B3HnvsMQCGDBnCNddcw+TJkwt0Jcuxxx5L165d2bVrF8899xwAF198Mc8//zyJiYl06NCB5s2bA1CvXj26detGmzZtOO+887jhhhsy93P99dczcuRI4uPjqVy5MtOmTcty9l0Szj33XFavXk2XLl2AyJulL7zwAn379mXKlCkkJCTQokULOnfuXKL9Hm0sr5dhmY3MqgBvA++6+6M5rI8D3nb3NsGbqbj7g8G6d4Hx7p7rsEz79u1dX9YhJW316tWcdtppBWtcSsMyIiUlp8ezmS119/Y5tc93WMYiA11/BVZHB7uZNYxqdiFw6O35mcAQM6tmZk2BZsDiQh2FiIgUS0GGZboBVwCfm1lKsGwsMNTMEokMuaQC1wK4+xdm9iqwisiVNjfoShkRkSOrIFfLzCfncfR/5bHNBGBCMeoSEZFi0CdURURCSOEuIhJC+iYmOXoVcIKwFZvSSreOXCQ0rlsm/Uo4KNzlqBE3ZlYhWm/Ot8XMG7sVqv+/PDqRGjWOIT19N2d06krnHkk5tnv/nVmcfMqpJDTO+TrvKVOmUKNGDa688kqSkpKYNGkS7dvneDXcYdLS0pgxYwbXX389EJnu9+abb9askCGkYRmRI+yGP43NNdgBPnh3Fl+v/TLHdZruVwpK4S5Sip6dPIkBvTowYugFpK5fC8D/G3U9c2a9CcDjD47nwrM6M+icbjxy3/8jZcknJM/5N49OuLvCTfcr5YuGZURKyaoVKbwz83VeeedDDmRkMOS8JFrFJ2au3/nTT7z/zizeTF6MmbFr505q16lD0jnn0bN3H/547bDMthVlul8pP3TmLlJKli1eyFl9+1O9eg1q1qpNr3POy7L+mFq1qFatGuNH38zcf79F9erVc93XkZru94orrgByn+43NjY2c7pfKd8U7iKlKK8pfytXrsyLb71H79+czwfvzuK6K3L/HtOKMt2vlB8Kd5FSckanrrz/ztvs27uXPem7mTf3nSzrf96Tzu7du+hx1rncNu5BvvzicwBq1KzJnmA63IKoSNP9ypGjMXc5aqRO7Jd1Qbbr3HObFbKo17mfFn86fc6/kEv69qRhoya07dgly/o96en84erL+M8vkel+R497AIC+Ay7i3ttv4Y0X/jd00/3KkVOgKX9Lm6b8lfwU7hr1iGcHNOSEX52S6/qEmA1Z7keH+4qDTQvdX0nTh5gkWolP+SsiIhWPwl1EJIQU7iIiIaRwFxEJIYW7iEgIKdxFREJI17nLUSPhf0/Oc330x3USCrC/Fb+vmB/BL+w0wcnJyUyaNIm33367lCsrumnTprFkyRKeeuqpLFMi5yQ5OZmqVavStWvXHNfPnDmTVatWMWbMGIYPH07//v0ZNCj3Tw9n98ADDzB27NjM+127dmXBggWFO6ASoHAXOQLcHXcnJkYvlkvbyJEj81yfnJxMzZo1cwz3jIwMBgwYwIABA4rcf/ZwL4tgBw3LiJSa7zZ+ywVndmLC2D9y6Xm9+H7zJqZNmcxv+53FoHO68cwjDwLw8897uHHYJQw+tzsXnd2Fd2a+DsC9995Lhw4daNOmDSNGjMic+yUpKYlRo0bRs2dPTjvtND799FMuuugimjVrxl133QVEZnts2bIlw4YNIyEhgUGDBvHzzz8fVuPs2bPp0qUL7dq1Y/DgwaQH0x688847tGzZku7du/P666/neHzTpk3jggsu4Pzzz6dp06Y89dRTPProo7Rt25bOnTvz448/ArB+/Xr69u3LGWecQY8ePVizZg0Ab731Fp06daJt27b07t2brVu3ApFZL6+66iqSkpI45ZRTmDx5co79/+1vf6N58+b06tWLjz/+OHN59HTHkydPplWrViQkJDBkyBBSU1OZMmUKjz32GImJiXz00UcMHz6cW2+9lTPPPJPbb7+dadOmceONN2bub+7cufTo0YPmzZtnvnrJ3qZ///4kJyczZswY9u7dS2JiIpdddhkANWvWBCJP8KNHj6ZNmzbEx8dnThuRnJxMUlISgwYNomXLllx22WV5zvNTUAp3kVKUun4t5w8awqvvzCN1/Tq+3fA1L779Hq+++xGrPk9h6aKPWZD8Hg1OaMhrs+fz+nsL6ZZ0NgA33ngjn376KStXrmTv3r1ZhkWqVq3KvHnzGDlyJAMHDuTpp59m5cqVTJs2jR07dgDw5ZdfMmLECFasWEHt2rWzfEkHwA8//MD999/P3LlzWbZsGe3bt+fRRx9l3759XHPNNbz11lt89NFHfP/997ke38qVK5kxYwaLFy/mzjvvpEaNGixfvpwuXbrw/PPPAzBixAiefPJJli5dyqRJkzK/Bap79+4sWrSI5cuXM2TIEB566KHM/a5Zs4Z3332XxYsXc88997B///4s/W7ZsoVx48bx8ccfM2fOHFatWpVjfRMnTmT58uWsWLGCKVOmEBcXx8iRIxk1ahQpKSn06NEDiMylM3fuXB555JHD/4apqXz44YfMmjWLkSNH5jkJ28SJE6levTopKSmZ8/Qc8vrrr5OSksJnn33G3LlzGT16NFu2bAFg+fLlPP7446xatYqvv/46y5NVUSncRUpRw8ZNSGjXAYCF8z5g4bz3ubRvT4ac14vUdWv5JvVrft2yFYvmJ/PYA+NY9skCatWuA8AHH3xAp06diI+P5/333+eLL77I3O+hYYP4+Hhat25Nw4YNqVatGqeccgobN24EoEmTJnTrFvkqwMsvv5z58+dnqW3RokWsWrWKbt26kZiYyPTp0/nmm29Ys2YNTZs2pVmzZpgZl19+ea7Hd+aZZ1KrVi0aNGhAnTp1OP/88zPrSk1NJT09nQULFjB48GASExO59tprMwNt06ZN9OnTh/j4eB5++OEsx9evXz+qVatG/fr1Of744zPP6g/55JNPSEpKokGDBlStWjXXKZETEhK47LLLeOGFF6hcOfdR6MGDB1OpUqUc111yySXExMTQrFkzTjnllMxXHoU1f/58hg4dSqVKlTjhhBPo1asXn376KQAdO3akcePGxMTEkJiYmGWe/aLSmLtIKapevUbmbXfnqhtGMfjy3x3W7uVZyXz0wWye+PO9dOl5Fr8beTPXX389S5YsoUmTJowfPz7LGeOhCb1iYmKyTO4VExOTOR1vTlMBR3N3zjnnHF566aUsy1NSUvKcqjha9r6j68rIyODgwYPUrVs388tBot10003ceuutDBgwgOTk5CxfQlKQKYYLUuOsWbOYN28eM2fO5L777svyBBItjFMq68xd5Ajp2uss/vnKi/y8JzKuvXXLZnb8sJ1t328htnp1+l90KcNG3MSazz/jl19+AaB+/fqkp6cX6XtOv/32WxYuXAjASy+9RPfu3bOs79y5Mx9//DHr1q0D4Oeff+arr76iZcuWbNiwgfXr12duW1S1a9emadOmvPbaa0Ak4D777DMAdu7cSaNGjQCYPn16ofbbqVMnkpOT2bFjB/v378/cf7SDBw+yceNGzjzzTB566CHS0tJIT0/Pc9rjnLz22mscPHiQ9evX8/XXX9OiRQvi4uJISUnJ7OPQ1xsCVKlS5bBhJIhMqfzKK69w4MABtm/fzrx58+jYsWOhjrswdOYuR43sly7mNStklu1KaIbIrr3OYsO6r7hi4LkA1DimJg888T98m/o1j024m5iYGCpXrsKdDzxC7Tp1uOaaa4iPjycuLo4OHToUur/TTjuN6dOnc+2119KsWTOuu+66LOsbNGjAtGnTGDp0aOaTyf3330/z5s2ZOnUq/fr1o379+nTv3j3Pr+7Lz4svvsh1113H/fffz/79+xkyZAinn34648ePZ/DgwTRq1IjOnTtnzjlfEA0bNmT8+PF06dKFhg0b0q5dOw4cOJClzYEDB7j88svZuXMn7s6oUaOoW7cu559/PoMGDeLNN9/kySefzLevFi1a0KtXL7Zu3cqUKVOIjY2lW7duNG3alPj4eNq0aUO7du0y248YMYKEhATatWuXZdz9wgsvZOHChZx++umYGQ899BAnnnhikYd58qMpf6VCONJT/kYrq+l/izPlb2pqKv379y9WKEv5oil/RURE4S4SRnFxcTprP8op3CW0HC+RD4OIlLWiPI4V7hJa36TtJ+PnXQp4qdDcnR07dhAbG1uo7fK9WsbMmgDPAycCB4Gp7v6EmR0HvALEAanAJe7+U7DNHcDVwAHgZnd/t1BViZSAJz/5iZuAk+v+gHH4NdGrbXuW+9+nR54EDu7J2nZrGT05rN5dvUz6lfInNjaWxo0bF2qbglwKmQH80d2XmVktYKmZzQGGA++5+0QzGwOMAW43s1bAEKA1cBIw18yau/uBXPYvUip2/XKQCfN25Lo+Nfa3We5fN20PAMnDs36g5bx9M0q+uAJIndivTPqVcMh3WMbdt7j7suD2bmA10AgYCBz65MF04ILg9kDgZXf/xd03AOuA0rtSX0REDlOoMXcziwPaAp8AJ7j7Fog8AQDHB80aARujNtsULBMRkSOkwOFuZjWBfwC3uPuuvJrmsOywQUszG2FmS8xsyfbt23PYREREiqpA4W5mVYgE+4vufmhy561m1jBY3xDYFizfBDSJ2rwxsDn7Pt19qru3d/f2DRo0KGr9IiKSg3zD3SJTov0VWO3uj0atmgkMC24PA96MWj7EzKqZWVOgGbAYERE5YgpytUw34ArgczNLCZaNBSYCr5rZ1cC3wGAAd//CzF4FVhG50uYGXSkjInJk5Rvu7j6fnMfRAc7OZZsJwIRi1CUiIsWgT6iKiISQwl1EJIQU7iIiIaRwFxEJIYW7iEgIKdxFREJI4S4iEkIF+RCTSKaifFG1iBx5OnMXEQkhhbuISAgp3EVEQkjhLiISQgp3EZEQUriLiISQwl1EJIQU7iIiIaRwFxEJIYW7iEgIKdxFREJI4S4iEkIKdxGREFK4i4iEkMJdRCSEFO4iIiGkcBcRCSGFu4hICCncRURCSOEuIhJCCncRkRBSuIuIhJDCXUQkhBTuIiIhlG+4m9lzZrbNzFZGLRtvZt+ZWUrw85uodXeY2Toz+9LM+pRW4SIikruCnLlPA/rmsPwxd08Mfv4FYGatgCFA62CbZ8ysUkkVKyIiBZNvuLv7PODHAu5vIPCyu//i7huAdUDHYtQnIiJFUJwx9xvNbEUwbHNssKwRsDGqzaZg2WHMbISZLTGzJdu3by9GGSIikl1Rw/0vwKlAIrAFeCRYbjm09Zx24O5T3b29u7dv0KBBEcsQEZGcFCnc3X2rux9w94PAs/x36GUT0CSqaWNgc/FKFBGRwipSuJtZw6i7FwKHrqSZCQwxs2pm1hRoBiwuXokiIlJYlfNrYGYvAUlAfTPbBIwDkswskciQSypwLYC7f2FmrwKrgAzgBnc/UCqVi4hIrvINd3cfmsPiv+bRfgIwoThFiYhI8egTqiIiIZTvmbuIlI24MbPKrO/Uif3KrG8pGTpzFxEJIYW7iEgIKdxFREJI4S4iEkIKdxGREFK4i4iEkMJdRCSEFO4iIiGkcBcRCSGFu4hICCncRURCSOEuIhJCCncRkRBSuIuIhJDCXUQkhBTuIiIhpHAXEQkhhbuISAgp3EVEQkjhLiISQgp3EZEQUriLiIRQ5bIuQORISo39bVmXIHJE6MxdRCSEFO4iIiGkcBcRCSGFu4hICCncRURCSOEuIhJC+Ya7mT1nZtvMbGXUsuPMbI6ZrQ3+PTZq3R1mts7MvjSzPqVVuIiI5K4gZ+7TgL7Zlo0B3nP3ZsB7wX3MrBUwBGgdbPOMmVUqsWpFRKRA8g13d58H/Jht8UBgenB7OnBB1PKX3f0Xd98ArAM6lkypIiJSUEUdcz/B3bcABP8eHyxvBGyMarcpWHYYMxthZkvMbMn27duLWIaIiOSkpN9QtRyWeU4N3X2qu7d39/YNGjQo4TJERI5uRQ33rWbWECD4d1uwfBPQJKpdY2Bz0csTEZGiKGq4zwSGBbeHAW9GLR9iZtXMrCnQDFhcvBJFRKSw8p0V0sxeApKA+ma2CRgHTAReNbOrgW+BwQDu/oWZvQqsAjKAG9z9QCnVLiIiucg33N19aC6rzs6l/QRgQnGKEhGR4tEnVEVEQkjhLiISQgp3EZEQUriLiISQwl1EJIQU7iIiIaRwFxEJoXyvcxc52qXG/jbL/bh9M8qoEpGC05m7iEgIKdxFREJI4S4iEkIKdxGREFK4i4iEkMJdRCSEFO4iIiGkcBcRCSGFu4hICCncRURCSNMPVEBxY2aVdQkiUs7pzF1EJIQU7iIiIaRwFxEJIYW7iEgIKdxFREJI4S4iEkIKdxGREFK4i4iEkMJdRCSEFO4iIiGkcBcRCSGFu4hICBVr4jAzSwV2AweADHdvb2bHAa8AcUAqcIm7/1S8MkVEpDBK4sz9THdPdPf2wf0xwHvu3gx4L7gvIiJHUGkMywwEpge3pwMXlEIfIiKSh+KGuwOzzWypmY0Ilp3g7lsAgn+PL2YfIiJSSMX9so5u7r7ZzI4H5pjZmoJuGDwZjAD41a9+VcwyRKQkldUXwqRO7Fcm/YZRscLd3TcH/24zszeAjsBWM2vo7lvMrCGwLZdtpwJTAdq3b+/FqUMkN6mxvy3rEkTKRJGHZczsGDOrdeg2cC6wEpgJDAuaDQPeLG6RIiJSOMU5cz8BeMPMDu1nhru/Y2afAq+a2dXAt8Dg4pcpIiKFUeRwd/evgdNzWL4DOLs4RYmISPHoE6oiIiGkcBcRCSGFu4hICCncRURCSOEuIhJCCncRkRBSuIuIhJDCXUQkhBTuIiIhpHAXEQmh4k75e1Qrq2lRRUTyozN3EZEQUriLiISQhmVEpNzQN0CVHJ25i4iEkM7cRQop+qv74vbNKMNKRHKnM3cRkRBSuIuIhJDCXUQkhBTuIiIhpHAXEQmhUFwto2kARESy0pm7iEgIKdxFREJI4S4iEkIKdxGREArFG6oi0aKnBxA5WincReSoV5ZX3JXWjJQKd5Fi0CRiUl5pzF1EJIQU7iIiIaRwFxEJoVILdzPra2Zfmtk6MxtTWv2IiMjhSuUNVTOrBDwNnANsAj41s5nuvqo0+hPR5Y8iWZXWmXtHYJ27f+3u/wFeBgaWUl8iIpJNaV0K2QjYGHV/E9ApuoGZjQBGBHfTzezLPPZXH/ihRCssO2E6Fignx2Mls5v6wA92z64ibt6/ZKooGeXi71KCwnQ8WY7F/lysfZ2c24rSCvec/q95ljvuU4GpBdqZ2RJ3b18ShZW1MB0LhOt4dCzlV5iO50gdS2kNy2wCmkTdbwxsLqW+REQkm9IK90+BZmbW1MyqAkOAmaXUl4iIZFMqwzLunmFmNwLvApWA59z9i2LsskDDNxVEmI4FwnU8OpbyK0zHc0SOxdw9/1YiIlKh6BOqIiIhpHAXEQmhChPuZnafma0wsxQzm21mJ5V1TUVlZg+b2ZrgeN4ws7plXVNRmdlgM/vCzA6aWYW8VC1MU2WY2XNmts3MVpZ1LcVlZk3M7AMzWx08xv5Q1jUVlZnFmtliM/ssOJZ7Sr3PijLmbma13X1XcPtmoJW7jyzjsorEzM4F3g/eeP4zgLvfXsZlFYmZnQYcBP4H+JO7LynjkgolmCrjK6KmygCGVtSpMsysJ5AOPO/ubcq6nuIws4ZAQ3dfZma1gKXABRXxb2NmBhzj7ulmVgWYD/zB3ReVVp8V5sz9ULAHjiHbh6IqEnef7e4Zwd1FRD4HUCG5+2p3z+vTxeVdqKbKcPd5wI9lXUdJcPct7r4suL0bWE3k0+8VjkekB3erBD+lmmEVJtwBzGyCmW0ELgPuLut6SshVwL/LuoijWE5TZVTIAAkzM4sD2gKflHEpRWZmlcwsBdgGzHH3Uj2WchXuZjbXzFbm8DMQwN3vdPcmwIvAjWVbbd7yO5agzZ1ABpHjKbcKciwVWL5TZUjZMrOawD+AW7K9gq9Q3P2AuycSeaXe0cxKddisXH2Hqrv3LmDTGcAsYFwpllMs+R2LmQ0jMtPU2V7O3/goxN+lItJUGeVYMD79D+BFd3+9rOspCe6eZmbJQF+g1N74Lldn7nkxs2ZRdwcAa8qqluIys77A7cAAd/+5rOs5ymmqjHIqeBPyr8Bqd3+0rOspDjNrcOiqODOrDvSmlDOsIl0t8w+gBZErM74BRrr7d2VbVdGY2TqgGrAjWLSoAl/5cyHwJNAASANS3L1PmRZVSGb2G+Bx/jtVxoSyrajozOwlIInItLJbgXHu/tcyLaqIzKw78BHwOZH/9wBj3f1fZVdV0ZhZAjCdyGMsBnjV3e8t1T4rSriLiEjBVZhhGRERKTiFu4hICCncRURCSOEuIhJCCncRkRBSuIuIhJDCXUQkhP4/E70d3kBcI0cAAAAASUVORK5CYII=\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAlVUlEQVR4nO3deXxU5dn/8c8VtoBsCqgI1KBlEUgMyL5GRcGC4AIKdYFqRVwrtiiiP8EFpYobLuXBpxZ8FLdWK0qrgBoRAZElIgIKSBQEAdEAQbAErt8fc0gnIftCksP3/Xrlxcw59zn3dZLhO2fuOXOPuTsiIhIuMWVdgIiIlDyFu4hICCncRURCSOEuIhJCCncRkRBSuIuIhJDCXY4oM/u3mQ07Av3EmZmbWeWS7tfMepjZl1H3U82sd0nsO9jfF2aWVFL7k6OT6Tr3cDOzIcAooA2wB9gATAf+4iH+45tZHJFjreLuGYXYzoFm7r6uENukAr9397lFqHMasMnd7yrstiJ50Zl7iJnZH4EngIeBE4ETgJFAN6BqGZZWYR16JSBS7rm7fkL4A9QhcqZ+cT7t+gHLgV3ARmB81LokImeV0e1Tgd7B7Y7AkmDbrcCjwfJY4AVgB5AGfAqcEKxLJnKWC3Aq8H7Q7gfgRaButr7+BKwAdgKvALG5HEclYFKwn6+BGwAHKufQ76+BD4N9/gC8EiyfF2yzB0gHLj30OwBuB74H/i/77yWo8w5gFfAT8LdDdQLDgfnZavWghhHAfuA/QX9v5fA7rgY8DmwOfh4HqkX/fYA/AtuALcDv8vhbJwP3AwsO9QfUC37vu4K/U1xU+5bAHOBH4EvgkgI+buKCYxwGfBv8ju8s6/8TR9uPztzDqwuRYHgzn3Z7gCuBukT+w15nZhcUsI8ngCfcvTaRoH41WD6MyJNLEyLhMRLYm8P2BjwInAScFrQfn63NJUBfoCmQQCQsc3IN0B9oC7QHBuVR933AbOBYoDHwJIC79wzWn+7uNd39leD+icBxwMlEAjknlwF9iPwemgP5DrO4+1QiwfpQ0N/5OTS7E+gMJAKnE3lCjd73iUR+142Aq4GnzezYPLodAlwRtD8VWEjkyeg4YDUwDsDMjiES7DOA44GhwDNm1jrYT0EeN92BFsDZwN1mdloedUkJU7iHV33gB48abzazBWaWZmZ7zawngLsnu/vn7n7Q3VcALwG9CtjHfuDXZlbf3dPdfVHU8nrAr939gLsvdfdd2Td293XuPsfdf3H37cCjOfQ92d03u/uPRM40E3Op5RLgcXffGLR9MJ+6TwZOcvd97j4/n+M8CIwL6szpSQrgqai+JxAJw5JwGXCvu28Lfkf3EAnnQ/YH6/e7+7+InJG3yGN/f3P39e6+E/g3sN7d5waPk9eIPDlC5Iky1d3/5u4Z7r4M+AfBk2YBHzf3uPted/8M+IzIk5McIQr38NoB1I8eI3b3ru5eN1gXA2BmnczsAzPbbmY7iZxl1y9gH1cTOUtdY2afmln/YPn/Ae8CL5vZZjN7yMyqZN/YzI43s5fN7Dsz20VkKCd7399H3f4ZqJlLLScRGR445Js86r6NyKuGxcGVKVfl0RZgu7vvy6dN9r5Pyqd9QZ1E1mPJvu8dnvUN47x+RxAZPjtkbw73D217MtApOBlIM7M0Ik80J0KBHzcF/dtJKVC4h9dC4BdgYD7tZgAzgSbuXgeYQiT4IPLSu8ahhmZWCWhw6L67r3X3oURetv8Z+LuZHROcRd7j7q2ArkTOAq/Moe8HiYzNJgRDO5dH9V1YW4gM6xzyq9wauvv37n6Nu58EXEtkuOHXeey7IFcVZe97c3A7++/wxELuezORoM1p36VpI/Chu9eN+qnp7tcF6/N63Eg5oHAPKXdPI/IS/hkzG2RmNc0sxswSgWOimtYCfnT3fWbWEfht1LqvgFgz6xeced9FZBwfADO73MwauPtBIm+cAhwwszPNLD54MthFZOjgQA5l1iIyjJBmZo2A0cU45FeBm82scTDmPCa3hmY22MwaB3d/IhKwh+rbCpxShP5vCPo+DhhL5M1fiAxHtDazRDOL5fD3FPLr7yXgLjNrYGb1gbuJvMIpbW8Dzc3sCjOrEvx0iBo3z+txI+WAwj3E3P0h4FYiwxDbiATJ/xC58mNB0Ox64F4z200kOF6N2n5nsP5/ge+InIVuiuqiL/CFmaUTeXN1SDB8cSLwdyLBvprIlSk5BdI9QDsiV63MAl4vxuE+S2Qo6DNgWT776gB8EtQ9E/iDu28I1o0HpgdDEZcUov8ZRN6k/Tr4uR/A3b8C7gXmAmuB7OP7fwVaBf39M4f93k/kiqQVwOfBsd1fiLqKxN13A+cSeQN2M5Ehlj/z3yf3XB83Uj7oQ0wiIiGkM3cRkRBSuIuIhJDCXUQkhBTuIiIhVC4mQapfv77HxcWVdRlylPvyy8gsvi1a5PUBT5HyY+nSpT+4e4Oc1pWLcI+Li2PJkiVlXYYc5ZKSkgBITk4u0zpECsrMcv0ktoZlRERCSOEuIhJCCncRkRAqF2PuIqVh//79bNq0iX378pvQMWLcuHEArF69ujTLEim02NhYGjduTJUqh02umiuFu4TWpk2bqFWrFnFxcZjlP2FhTEzkhayulpHyxN3ZsWMHmzZtomnTpgXeTsMyElr79u2jXr16BQp2kfLKzKhXr16BX4EeonCXUFOwSxgU5XGscBcRCSGFu8gRMn78eCZNmgTA3Xffzdy5c3Nt+89//pNVq1blun7KlCk8//zzQOTDV4X5EGBaWhrPPPNM5v3NmzczaFBe3ycuFZHCXQRg83L4ZXfkZ/PyUu/u3nvvpXfv3rmuzyvcMzIyGDlyJFdemdM3F+Yve7ifdNJJ/P3vfy/SvqT8UriLlKIJEybQokULevfunTl3DcDw4cMzA3XMmDG0atWKhIQE/vSnP7FgwQJmzpzJ6NGjSUxMZP369SQlJTF27Fh69erFE088keVVAMALL7xA165dadOmDYsXLwY4rE2bNm1ITU1lzJgxrF+/nsTEREaPHk1qaipt2rQBIm9C/+53vyM+Pp62bdvywQcfADBt2jQuuugi+vbtS7NmzbjttttyPN64uDjGjh1Lly5daN++PcuWLaNPnz6ceuqpTJkyJbPdww8/TIcOHUhISMi8BBXgggsu4IwzzqB169ZMnTo1c3nNmjW58847Of300+ncuTNbt25F8pbvpZBm1gR4nshXpx0Eprr7E2Y2HrgG2B40Hevu/wq2uQO4msj3Ut7s7u+WQu0iBXbLLbeQkpKSe4P/pPPz/si3ktWoYlC1Zr77TExM5PHHH891/dKlS3n55ZdZvnw5GRkZtGvXjjPOOCNLmx9//JE33niDNWvWYGakpaVRt25dBgwYQP/+/bMMl6SlpfHhhx8CkeCOtmfPHhYsWMC8efO46qqrWLlyZa51TZw4kZUrV2b+PlJTUzPXPf300wB8/vnnrFmzhnPPPZevvvoKgJSUFJYvX061atVo0aIFN910E02aNMm+e5o0acLChQsZNWoUw4cP5+OPP2bfvn20bt2akSNHMnv2bNauXcvixYtxdwYMGMC8efPo2bMnzz33HMcddxx79+6lQ4cOXHzxxdSrV489e/bQuXNnJkyYwG233cazzz7LXXfdlesxSsGuc88A/ujuy8ysFrDUzOYE6x5z90nRjc2sFZHvXWwNnATMNbPm7p7TFySLhNZHH33EhRdeSI0aNQAYMGDAYW1q165NbGwsv//97+nXrx/9+/fPdX+XXnppruuGDh0KQM+ePdm1axdpaWlFqnn+/PncdNNNALRs2ZKTTz45M9zPPvts6tSpA0CrVq345ptvcgz3Q8cZHx9Peno6tWrVolatWsTGxpKWlsbs2bOZPXs2bdu2BSA9PZ21a9fSs2dPJk+ezBtvvAHAxo0bWbt2LfXq1aNq1aqZv5szzjiDOXPmHNavZJVvuLv7FmBLcHu3ma0GGuWxyUDgZXf/BdhgZuuAjsDCEqhXpEjyOsMGYPNyvvwhcv7Ron4lOKltifSb3yVslStXZvHixbz33nu8/PLLPPXUU7z//vs5tj3mmGMK3I+ZUblyZQ4ePJi5rCDXSef1ncrVqlXLvF2pUiUyMjLybBcTE5Nlm5iYGDIyMnB37rjjDq699tos2yUnJzN37lwWLlxIjRo1SEpKyqy5SpUqmceYV9/yX4UaczezOKAt8Emw6EYzW2Fmz5nZscGyRsDGqM02kcOTgZmNMLMlZrZk+/bt2VeLVHg9e/bkjTfeYO/evezevZu33nrrsDbp6ens3LmT3/zmNzz++OOZQyW1atVi9+7dBe7rlVdeASJn3nXq1KFOnTrExcWxbNkyAJYtW8aGDRvy3XfPnj158cUXAfjqq6/49ttvS/wTu3369OG5554jPT0dgO+++45t27axc+dOjj32WGrUqMGaNWtYtGhRifZ7tCnw9ANmVhP4B3CLu+8ys78A9wEe/PsIcBWQ06nKYacD7j4VmArQvn373E8XRCqodu3acemll5KYmMjJJ59Mjx49Dmuze/duBg4cyL59+3B3HnvsMQCGDBnCNddcw+TJkwt0Jcuxxx5L165d2bVrF8899xwAF198Mc8//zyJiYl06NCB5s2bA1CvXj26detGmzZtOO+887jhhhsy93P99dczcuRI4uPjqVy5MtOmTcty9l0Szj33XFavXk2XLl2AyJulL7zwAn379mXKlCkkJCTQokULOnfuXKL9Hm0sr5dhmY3MqgBvA++6+6M5rI8D3nb3NsGbqbj7g8G6d4Hx7p7rsEz79u1dX9YhJW316tWcdtppBWtcSsMyIiUlp8ezmS119/Y5tc93WMYiA11/BVZHB7uZNYxqdiFw6O35mcAQM6tmZk2BZsDiQh2FiIgUS0GGZboBVwCfm1lKsGwsMNTMEokMuaQC1wK4+xdm9iqwisiVNjfoShkRkSOrIFfLzCfncfR/5bHNBGBCMeoSEZFi0CdURURCSOEuIhJC+iYmOXoVcIKwFZvSSreOXCQ0rlsm/Uo4KNzlqBE3ZlYhWm/Ot8XMG7sVqv+/PDqRGjWOIT19N2d06krnHkk5tnv/nVmcfMqpJDTO+TrvKVOmUKNGDa688kqSkpKYNGkS7dvneDXcYdLS0pgxYwbXX389EJnu9+abb9askCGkYRmRI+yGP43NNdgBPnh3Fl+v/TLHdZruVwpK4S5Sip6dPIkBvTowYugFpK5fC8D/G3U9c2a9CcDjD47nwrM6M+icbjxy3/8jZcknJM/5N49OuLvCTfcr5YuGZURKyaoVKbwz83VeeedDDmRkMOS8JFrFJ2au3/nTT7z/zizeTF6MmbFr505q16lD0jnn0bN3H/547bDMthVlul8pP3TmLlJKli1eyFl9+1O9eg1q1qpNr3POy7L+mFq1qFatGuNH38zcf79F9erVc93XkZru94orrgByn+43NjY2c7pfKd8U7iKlKK8pfytXrsyLb71H79+czwfvzuK6K3L/HtOKMt2vlB8Kd5FSckanrrz/ztvs27uXPem7mTf3nSzrf96Tzu7du+hx1rncNu5BvvzicwBq1KzJnmA63IKoSNP9ypGjMXc5aqRO7Jd1Qbbr3HObFbKo17mfFn86fc6/kEv69qRhoya07dgly/o96en84erL+M8vkel+R497AIC+Ay7i3ttv4Y0X/jd00/3KkVOgKX9Lm6b8lfwU7hr1iGcHNOSEX52S6/qEmA1Z7keH+4qDTQvdX0nTh5gkWolP+SsiIhWPwl1EJIQU7iIiIaRwFxEJIYW7iEgIKdxFREJI17nLUSPhf0/Oc330x3USCrC/Fb+vmB/BL+w0wcnJyUyaNIm33367lCsrumnTprFkyRKeeuqpLFMi5yQ5OZmqVavStWvXHNfPnDmTVatWMWbMGIYPH07//v0ZNCj3Tw9n98ADDzB27NjM+127dmXBggWFO6ASoHAXOQLcHXcnJkYvlkvbyJEj81yfnJxMzZo1cwz3jIwMBgwYwIABA4rcf/ZwL4tgBw3LiJSa7zZ+ywVndmLC2D9y6Xm9+H7zJqZNmcxv+53FoHO68cwjDwLw8897uHHYJQw+tzsXnd2Fd2a+DsC9995Lhw4daNOmDSNGjMic+yUpKYlRo0bRs2dPTjvtND799FMuuugimjVrxl133QVEZnts2bIlw4YNIyEhgUGDBvHzzz8fVuPs2bPp0qUL7dq1Y/DgwaQH0x688847tGzZku7du/P666/neHzTpk3jggsu4Pzzz6dp06Y89dRTPProo7Rt25bOnTvz448/ArB+/Xr69u3LGWecQY8ePVizZg0Ab731Fp06daJt27b07t2brVu3ApFZL6+66iqSkpI45ZRTmDx5co79/+1vf6N58+b06tWLjz/+OHN59HTHkydPplWrViQkJDBkyBBSU1OZMmUKjz32GImJiXz00UcMHz6cW2+9lTPPPJPbb7+dadOmceONN2bub+7cufTo0YPmzZtnvnrJ3qZ///4kJyczZswY9u7dS2JiIpdddhkANWvWBCJP8KNHj6ZNmzbEx8dnThuRnJxMUlISgwYNomXLllx22WV5zvNTUAp3kVKUun4t5w8awqvvzCN1/Tq+3fA1L779Hq+++xGrPk9h6aKPWZD8Hg1OaMhrs+fz+nsL6ZZ0NgA33ngjn376KStXrmTv3r1ZhkWqVq3KvHnzGDlyJAMHDuTpp59m5cqVTJs2jR07dgDw5ZdfMmLECFasWEHt2rWzfEkHwA8//MD999/P3LlzWbZsGe3bt+fRRx9l3759XHPNNbz11lt89NFHfP/997ke38qVK5kxYwaLFy/mzjvvpEaNGixfvpwuXbrw/PPPAzBixAiefPJJli5dyqRJkzK/Bap79+4sWrSI5cuXM2TIEB566KHM/a5Zs4Z3332XxYsXc88997B///4s/W7ZsoVx48bx8ccfM2fOHFatWpVjfRMnTmT58uWsWLGCKVOmEBcXx8iRIxk1ahQpKSn06NEDiMylM3fuXB555JHD/4apqXz44YfMmjWLkSNH5jkJ28SJE6levTopKSmZ8/Qc8vrrr5OSksJnn33G3LlzGT16NFu2bAFg+fLlPP7446xatYqvv/46y5NVUSncRUpRw8ZNSGjXAYCF8z5g4bz3ubRvT4ac14vUdWv5JvVrft2yFYvmJ/PYA+NY9skCatWuA8AHH3xAp06diI+P5/333+eLL77I3O+hYYP4+Hhat25Nw4YNqVatGqeccgobN24EoEmTJnTrFvkqwMsvv5z58+dnqW3RokWsWrWKbt26kZiYyPTp0/nmm29Ys2YNTZs2pVmzZpgZl19+ea7Hd+aZZ1KrVi0aNGhAnTp1OP/88zPrSk1NJT09nQULFjB48GASExO59tprMwNt06ZN9OnTh/j4eB5++OEsx9evXz+qVatG/fr1Of744zPP6g/55JNPSEpKokGDBlStWjXXKZETEhK47LLLeOGFF6hcOfdR6MGDB1OpUqUc111yySXExMTQrFkzTjnllMxXHoU1f/58hg4dSqVKlTjhhBPo1asXn376KQAdO3akcePGxMTEkJiYmGWe/aLSmLtIKapevUbmbXfnqhtGMfjy3x3W7uVZyXz0wWye+PO9dOl5Fr8beTPXX389S5YsoUmTJowfPz7LGeOhCb1iYmKyTO4VExOTOR1vTlMBR3N3zjnnHF566aUsy1NSUvKcqjha9r6j68rIyODgwYPUrVs388tBot10003ceuutDBgwgOTk5CxfQlKQKYYLUuOsWbOYN28eM2fO5L777svyBBItjFMq68xd5Ajp2uss/vnKi/y8JzKuvXXLZnb8sJ1t328htnp1+l90KcNG3MSazz/jl19+AaB+/fqkp6cX6XtOv/32WxYuXAjASy+9RPfu3bOs79y5Mx9//DHr1q0D4Oeff+arr76iZcuWbNiwgfXr12duW1S1a9emadOmvPbaa0Ak4D777DMAdu7cSaNGjQCYPn16ofbbqVMnkpOT2bFjB/v378/cf7SDBw+yceNGzjzzTB566CHS0tJIT0/Pc9rjnLz22mscPHiQ9evX8/XXX9OiRQvi4uJISUnJ7OPQ1xsCVKlS5bBhJIhMqfzKK69w4MABtm/fzrx58+jYsWOhjrswdOYuR43sly7mNStklu1KaIbIrr3OYsO6r7hi4LkA1DimJg888T98m/o1j024m5iYGCpXrsKdDzxC7Tp1uOaaa4iPjycuLo4OHToUur/TTjuN6dOnc+2119KsWTOuu+66LOsbNGjAtGnTGDp0aOaTyf3330/z5s2ZOnUq/fr1o379+nTv3j3Pr+7Lz4svvsh1113H/fffz/79+xkyZAinn34648ePZ/DgwTRq1IjOnTtnzjlfEA0bNmT8+PF06dKFhg0b0q5dOw4cOJClzYEDB7j88svZuXMn7s6oUaOoW7cu559/PoMGDeLNN9/kySefzLevFi1a0KtXL7Zu3cqUKVOIjY2lW7duNG3alPj4eNq0aUO7du0y248YMYKEhATatWuXZdz9wgsvZOHChZx++umYGQ899BAnnnhikYd58qMpf6VCONJT/kYrq+l/izPlb2pqKv379y9WKEv5oil/RURE4S4SRnFxcTprP8op3CW0HC+RD4OIlLWiPI4V7hJa36TtJ+PnXQp4qdDcnR07dhAbG1uo7fK9WsbMmgDPAycCB4Gp7v6EmR0HvALEAanAJe7+U7DNHcDVwAHgZnd/t1BViZSAJz/5iZuAk+v+gHH4NdGrbXuW+9+nR54EDu7J2nZrGT05rN5dvUz6lfInNjaWxo0bF2qbglwKmQH80d2XmVktYKmZzQGGA++5+0QzGwOMAW43s1bAEKA1cBIw18yau/uBXPYvUip2/XKQCfN25Lo+Nfa3We5fN20PAMnDs36g5bx9M0q+uAJIndivTPqVcMh3WMbdt7j7suD2bmA10AgYCBz65MF04ILg9kDgZXf/xd03AOuA0rtSX0REDlOoMXcziwPaAp8AJ7j7Fog8AQDHB80aARujNtsULBMRkSOkwOFuZjWBfwC3uPuuvJrmsOywQUszG2FmS8xsyfbt23PYREREiqpA4W5mVYgE+4vufmhy561m1jBY3xDYFizfBDSJ2rwxsDn7Pt19qru3d/f2DRo0KGr9IiKSg3zD3SJTov0VWO3uj0atmgkMC24PA96MWj7EzKqZWVOgGbAYERE5YgpytUw34ArgczNLCZaNBSYCr5rZ1cC3wGAAd//CzF4FVhG50uYGXSkjInJk5Rvu7j6fnMfRAc7OZZsJwIRi1CUiIsWgT6iKiISQwl1EJIQU7iIiIaRwFxEJIYW7iEgIKdxFREJI4S4iEkIF+RCTSKaifFG1iBx5OnMXEQkhhbuISAgp3EVEQkjhLiISQgp3EZEQUriLiISQwl1EJIQU7iIiIaRwFxEJIYW7iEgIKdxFREJI4S4iEkIKdxGREFK4i4iEkMJdRCSEFO4iIiGkcBcRCSGFu4hICCncRURCSOEuIhJCCncRkRBSuIuIhJDCXUQkhBTuIiIhlG+4m9lzZrbNzFZGLRtvZt+ZWUrw85uodXeY2Toz+9LM+pRW4SIikruCnLlPA/rmsPwxd08Mfv4FYGatgCFA62CbZ8ysUkkVKyIiBZNvuLv7PODHAu5vIPCyu//i7huAdUDHYtQnIiJFUJwx9xvNbEUwbHNssKwRsDGqzaZg2WHMbISZLTGzJdu3by9GGSIikl1Rw/0vwKlAIrAFeCRYbjm09Zx24O5T3b29u7dv0KBBEcsQEZGcFCnc3X2rux9w94PAs/x36GUT0CSqaWNgc/FKFBGRwipSuJtZw6i7FwKHrqSZCQwxs2pm1hRoBiwuXokiIlJYlfNrYGYvAUlAfTPbBIwDkswskciQSypwLYC7f2FmrwKrgAzgBnc/UCqVi4hIrvINd3cfmsPiv+bRfgIwoThFiYhI8egTqiIiIZTvmbuIlI24MbPKrO/Uif3KrG8pGTpzFxEJIYW7iEgIKdxFREJI4S4iEkIKdxGREFK4i4iEkMJdRCSEFO4iIiGkcBcRCSGFu4hICCncRURCSOEuIhJCCncRkRBSuIuIhJDCXUQkhBTuIiIhpHAXEQkhhbuISAgp3EVEQkjhLiISQgp3EZEQUriLiIRQ5bIuQORISo39bVmXIHJE6MxdRCSEFO4iIiGkcBcRCSGFu4hICCncRURCSOEuIhJC+Ya7mT1nZtvMbGXUsuPMbI6ZrQ3+PTZq3R1mts7MvjSzPqVVuIiI5K4gZ+7TgL7Zlo0B3nP3ZsB7wX3MrBUwBGgdbPOMmVUqsWpFRKRA8g13d58H/Jht8UBgenB7OnBB1PKX3f0Xd98ArAM6lkypIiJSUEUdcz/B3bcABP8eHyxvBGyMarcpWHYYMxthZkvMbMn27duLWIaIiOSkpN9QtRyWeU4N3X2qu7d39/YNGjQo4TJERI5uRQ33rWbWECD4d1uwfBPQJKpdY2Bz0csTEZGiKGq4zwSGBbeHAW9GLR9iZtXMrCnQDFhcvBJFRKSw8p0V0sxeApKA+ma2CRgHTAReNbOrgW+BwQDu/oWZvQqsAjKAG9z9QCnVLiIiucg33N19aC6rzs6l/QRgQnGKEhGR4tEnVEVEQkjhLiISQgp3EZEQUriLiISQwl1EJIQU7iIiIaRwFxEJoXyvcxc52qXG/jbL/bh9M8qoEpGC05m7iEgIKdxFREJI4S4iEkIKdxGREFK4i4iEkMJdRCSEFO4iIiGkcBcRCSGFu4hICCncRURCSNMPVEBxY2aVdQkiUs7pzF1EJIQU7iIiIaRwFxEJIYW7iEgIKdxFREJI4S4iEkIKdxGREFK4i4iEkMJdRCSEFO4iIiGkcBcRCSGFu4hICBVr4jAzSwV2AweADHdvb2bHAa8AcUAqcIm7/1S8MkVEpDBK4sz9THdPdPf2wf0xwHvu3gx4L7gvIiJHUGkMywwEpge3pwMXlEIfIiKSh+KGuwOzzWypmY0Ilp3g7lsAgn+PL2YfIiJSSMX9so5u7r7ZzI4H5pjZmoJuGDwZjAD41a9+VcwyRKQkldUXwqRO7Fcm/YZRscLd3TcH/24zszeAjsBWM2vo7lvMrCGwLZdtpwJTAdq3b+/FqUMkN6mxvy3rEkTKRJGHZczsGDOrdeg2cC6wEpgJDAuaDQPeLG6RIiJSOMU5cz8BeMPMDu1nhru/Y2afAq+a2dXAt8Dg4pcpIiKFUeRwd/evgdNzWL4DOLs4RYmISPHoE6oiIiGkcBcRCSGFu4hICCncRURCSOEuIhJCCncRkRBSuIuIhJDCXUQkhBTuIiIhpHAXEQmh4k75e1Qrq2lRRUTyozN3EZEQUriLiISQhmVEpNzQN0CVHJ25i4iEkM7cRQop+qv74vbNKMNKRHKnM3cRkRBSuIuIhJDCXUQkhBTuIiIhpHAXEQmhUFwto2kARESy0pm7iEgIKdxFREJI4S4iEkIKdxGREArFG6oi0aKnBxA5WincReSoV5ZX3JXWjJQKd5Fi0CRiUl5pzF1EJIQU7iIiIaRwFxEJoVILdzPra2Zfmtk6MxtTWv2IiMjhSuUNVTOrBDwNnANsAj41s5nuvqo0+hPR5Y8iWZXWmXtHYJ27f+3u/wFeBgaWUl8iIpJNaV0K2QjYGHV/E9ApuoGZjQBGBHfTzezLPPZXH/ihRCssO2E6Fignx2Mls5v6wA92z64ibt6/ZKooGeXi71KCwnQ8WY7F/lysfZ2c24rSCvec/q95ljvuU4GpBdqZ2RJ3b18ShZW1MB0LhOt4dCzlV5iO50gdS2kNy2wCmkTdbwxsLqW+REQkm9IK90+BZmbW1MyqAkOAmaXUl4iIZFMqwzLunmFmNwLvApWA59z9i2LsskDDNxVEmI4FwnU8OpbyK0zHc0SOxdw9/1YiIlKh6BOqIiIhpHAXEQmhChPuZnafma0wsxQzm21mJ5V1TUVlZg+b2ZrgeN4ws7plXVNRmdlgM/vCzA6aWYW8VC1MU2WY2XNmts3MVpZ1LcVlZk3M7AMzWx08xv5Q1jUVlZnFmtliM/ssOJZ7Sr3PijLmbma13X1XcPtmoJW7jyzjsorEzM4F3g/eeP4zgLvfXsZlFYmZnQYcBP4H+JO7LynjkgolmCrjK6KmygCGVtSpMsysJ5AOPO/ubcq6nuIws4ZAQ3dfZma1gKXABRXxb2NmBhzj7ulmVgWYD/zB3ReVVp8V5sz9ULAHjiHbh6IqEnef7e4Zwd1FRD4HUCG5+2p3z+vTxeVdqKbKcPd5wI9lXUdJcPct7r4suL0bWE3k0+8VjkekB3erBD+lmmEVJtwBzGyCmW0ELgPuLut6SshVwL/LuoijWE5TZVTIAAkzM4sD2gKflHEpRWZmlcwsBdgGzHH3Uj2WchXuZjbXzFbm8DMQwN3vdPcmwIvAjWVbbd7yO5agzZ1ABpHjKbcKciwVWL5TZUjZMrOawD+AW7K9gq9Q3P2AuycSeaXe0cxKddisXH2Hqrv3LmDTGcAsYFwpllMs+R2LmQ0jMtPU2V7O3/goxN+lItJUGeVYMD79D+BFd3+9rOspCe6eZmbJQF+g1N74Lldn7nkxs2ZRdwcAa8qqluIys77A7cAAd/+5rOs5ymmqjHIqeBPyr8Bqd3+0rOspDjNrcOiqODOrDvSmlDOsIl0t8w+gBZErM74BRrr7d2VbVdGY2TqgGrAjWLSoAl/5cyHwJNAASANS3L1PmRZVSGb2G+Bx/jtVxoSyrajozOwlIInItLJbgXHu/tcyLaqIzKw78BHwOZH/9wBj3f1fZVdV0ZhZAjCdyGMsBnjV3e8t1T4rSriLiEjBVZhhGRERKTiFu4hICCncRURCSOEuIhJCCncRkRBSuIuIhJDCXUQkhP4/E70d3kBcI0cAAAAASUVORK5CYII=\n", "text/plain": [ "
    " ] @@ -2103,7 +1723,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 48, @@ -2112,7 +1732,7 @@ }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAkgUlEQVR4nO3deXhU5d3/8feXRUAFRUEBQQMWBCQQfSJiWYygIoIiFCq4sViBVrTQ9vcUbVWKG7Yo1V+xfdB6BZ4iiCsuLVjRFFApBhtZBBUlatiJGyggy/f545yESZgkkx0On9d1zZU5+31OZj5zn3vO3MfcHRERiZYa1V0AERGpeAp3EZEIUriLiESQwl1EJIIU7iIiEaRwFxGJIIX7EcrMJprZ36q7HEcDMzvdzHaaWc0q2Fb+/7Wit2tmfzGzO8LnaWaWUxHrDdfX3cw+qKj1VRYzyzCzn1R3OaqCwj0BZpZtZrvCN9pmM0s3s+NjpqebmZvZlYWW+2M4fng4fIyZPWhmOeG61pvZ1CK2k/f4UyXsjz4YSsHdP3P34919/+G4XTMbbmZLEljfGHe/uyLKFr6ufxCz7sXuflZFrLs6mdn48D3+tZk9YWZ1qrtMZaVwT9wV7n48kAKcA9xWaPqHwLC8ATOrBQwGPo6Z5zYgFegM1AcuAv4Tbzsxj7EVuhdyVKqKs44jnZn1BiYAvYAkoBXwu+osU3ko3EvJ3TcDCwhCPtZLQFczaxgOXwasADbHzHMe8Ly7b/RAtrvPLEdx6prZU2a2w8zeNbNOeRPMrJmZPWtm28IzhFvD8ZcBtwNXh2cG75nZRWa2MmbZ18xsWczwEjO7qrj1htNqmNkEM/vYzHLNbK6ZnRROSwpre8PM7DMz225mvylqx8zsZDN7ycy+MbN3zOye2NqpmT1sZp+H05ebWfeYaelmdk/McIEmCDP7tZltCI/bB2bWKxzf2cwyw3VuMbOHCpW9Vjg8wszWhMt/YmajC2/LzH5pZlvNbJOZjShmP1ua2b/Cdf0TaBQzrfB2h4fb2xEe+2vNrB3wF+CC8P/5Vcwx+LOZ/d3MvgUuKnxcwvluD/8X2WZ2bcz4As0XsWcHZrYoHP1euM2r4xzjduE6vjKz1RZzVhuWY5qZvRLuy7/N7MxijtHTdrA2vcjMzk50XWZ2iZmtDZf9E2BFbYegcvZXd1/t7l8CdwPDi5n/sKZwLyUzaw70AdYVmrQbeBEYEg7fABQO7qXAL8zsZ2aWbGbFvdAS0R94GjgJeBJ4wcxqm1kNgg+b94DTCGoi48yst7vPB+4DngrPDDoBbwM/MLNGYZB0AJqbWX0zqwf8F7C4uPWG5bkVuAq4EGgGfAlMK1TmbsBZ4bJ3huEUzzTgW6AJwZtuWKHp7xB8wObt+9NmVrekA2ZmZwFjgfPcvT7QG8gOJz8MPOzuDYAzgblFrGYr0A9oAIwApprZuTHTmwAnEByjG4FpdvBDv7AngeUEoX53nP3MK/dxwCNAn7DcPwSy3H0NMAZ4O/x/nhiz2DXAvQRnifGabZqE2z0t3O708PgUy917hE87hdt8qlBZaxO8Tl4FTgFuAWYVWvdQglpxQ4L30r3FbPIfQOtwXe8CswpNj7suM2sEPAv8NtzPj4GuxWznbILXdp73gFPN7ORiljlsKdwT94KZ7QA+J3hz3xVnnpnADWZ2AkHAvVBo+v3AA8C1QCawwcwKv5lfCGs7eY+biinTcnd/xt33Ag8BdYEuBGcIjd19krt/7+6fAI9x8IOnAHffHZanB0Gz0QqCMOgaru8jd89NYL2jgd+4e4677wEmAoPyap6h37n7Lnd/j+DN04lCLGhC+BFwl7t/5+7vAzMKlflv7p7r7vvc/UGgDsGHRkn2h/O2N7Pa4dlTXtPZXsIPOXff6e5Lizher7j7x+HZ178IQqx7zCx7gUnuvtfd/w7sjFc2Mzud4Jje4e573H0RQSgW5QDQwczqufsmd19dwr7Oc/c33f1A+D+OJ2/b/wJeAX5cwjoT0QU4Hpgcvk5eB14mCOE8z7n7MnffRxDWKUWtzN2fcPcdMa+pTuF7rKR1XQ68H/Me+SMFz6QLOx74OmY473n94nb2cKVwT9xVYY0pDWhLzOlzHndfAjQmqCm87O67Ck3f7+7T3L0rcCJBDeOJQrXXq9z9xJjHY8WU6fOYdR8AcghqzGcAzWI/JAiaYk4tZl3/CvetR/g8g+AD6sJwmATWewbwfMy0NQRhGrvd2DfXdwRvqMIaA7Vi96/Qc8JmjzXh6fZXBDXlQ/4nhbn7OmAcQUhsNbM5ZtYsnHwj0AZYa0FTUL946zCzPma21My+CLd9eaFt54ZBU9J+NgO+dPdvY8Z9WkS5vwWuJqilbwqbIdoWv7cFj1kc8bbdrKiZS6EZ8Hn4moxd92kxw4m8DjCzmmY22YKmvm84eJYVe7yLWlczCr5HnOKPyU6Cs7E8ec93FLPMYUvhXkphDScdmFLELH8DfsmhTTKF17PL3acRNF20L2NxWuQ9CZtMmgMbCV7A6wt9SNR398vzNh9nXYXD/V8cGu4lrfdzgmaD2Ol13X1DKfdrG7Av3J94+9od+DVBLbNh2BTxNQfbU78Fjo1Ztknsyt39SXfvRvBh5ARnU7j7R+4+lOD0/wHgmbA5JJ8FV088S/D/PzXc9t8pvi23KJuAhoW2cXpRM7v7Ane/BGgKrCU4a4L4/8/ixueJt+2N4fNij2EJNgItwtdk7LpL+zqAoGmpP3AxwQd4Ujg+keO9iYKvG4sdjmM1Bc8kOwFbwrPWI47CvWz+CFxiZilxpj0CXAIsKjzBzMaFXzzVM7NaYZNMfQ69YiZR/2VmA8Nmj3HAHoJ2/WXANxZ8cVgvrP10MLPzwuW2AEmF3nxvETQddAaWhaf8ZwDnx+xLSev9C3CvmZ0R7m9jM+tf2p3y4NK/54CJZnZsWEO9IWaW+gThvw2oZWZ3UrDGlQVcbmYnmVmT8NgQluksM+sZhvRuYBfB2QVmdp2ZNQ5rnF+FixS+DPEYgmadbcA+M+sDXFrafQz381OC5rDfWXCZbDfginjzmtmpZnZlGMZ7CGqZeWXbQvAdyTFlKEbetrsTfI/wdDg+CxgYHv8fEJzVxNpCcDVJPP8m+HD47/A7oLRwv+aUoXz1CfY3l+DD5r5SLPsKcHbMe+RWiv+QmgncaGbtw+9IfktQkTsiKdzLwN23EbwQ7ogz7Qt3XxieAha2C3iQ4DRyO3Az8KOw7TrPS1bwOvfniynKPIJT9S+B64GBYTvvfoI3UwqwPtzW4wQ1Hzj4Bs41s3fDcn9L8GXVanf/Ppz+NvCpu28N5ylpvQ8TfKn8avj9xFKCD4eyGBuudzPwv8Bsgjc5BFcr/YPg8tNPCUI69nT7fwna87MJ2sNjv/CrA0wOy76ZoJZ+ezjtMmC1me0M92VI4bZqd99BEBJzCY77NeE+l9U1BMfoC4LvcYo646tBcEa4MZz3QuBn4bTXCWqdm81seym2vZlgHzYStFWPcfe14bSpwPcEIT6DQ7/EnAjMCJvgCrTTh6+fKwkuPNgOPArcELPu0phJ8D/eALxP8JpKiLtvJ7gceTLBh0Nr4M1i5p8P/B54I9zmp8T/bu2IYPEzSOTwYmYPAE3cPe7VJCJSkGruclgys7Zm1tECnQmaBYo7ixGRGLVKnkWkWtQnaIppRnDp6YMEzVAikgA1y4iIRJCaZUREIuiwaJZp1KiRJyUlVXcxRCrEBx8EPd+eddYR30miHOaWL1++3d0bx5t2WIR7UlISmZmZ1V0MkQqRlpYGQEZGRrWWQ6LPzOL+ohnULCMiEkkKdxGRCCox3M2shZm9EXbStNrMfh6On2hBn9hZ4ePymGVuM7N1FvSV3bvotYuISGVIpM19H/BLd3/XzOoDyy24qQDAVHcv0IGWmbUn6AL2bIJrlF8zszZexbcoE9m7dy85OTns3l1Ub7eV4667gl+sr1mzpkq3K9FVt25dmjdvTu3atRNepsRwd/dNBL2r4e47zGwNBbvuLKw/MCfse3m9ma0j6Izq7YRLJVIBcnJyqF+/PklJSVi574uSuBo1ghNiXS0jFcHdyc3NJScnh5YtWya8XKna3M0sieD+of8OR401sxUW3Eg2704zp1GwE6cc4nwYmNkoC25plrlt27bSFEMkIbt37+bkk0+u0mAXqWhmxsknn1zqM9CEw93Mjifox3qcu38D/JngVmQpBDX7B/NmjbP4IT+Ddffp7p7q7qmNG8e9TFOk3BTsEgVleR0nFO4W3BPxWWCWuz8H4O5bwjsLHSC4aUDncPYcCnaIn3cDCRERqSKJXC1jwF+BNe7+UMz4pjGzDQBWhc9fBIaYWR0za0nQh/KyiiuyyJGjZs2apKSk0KFDB6644gq++uorALKzszEz7rjj4C0Btm/fTu3atRk7diwQ/NI1LS2NlJQU2rVrx6hRo4Dgx1EnnHACKSkp+Y/XXnutyvdNDm+JXC3TleBGECvNLCscdzswNLwTkRPcFGE0gLuvNrO5BB3r7wNu1pUyciRbkfNVqebfu2df/nJ16tZj5ssZAPx2/E+5474p3HTrr9iw6Ruan57E08/PY/DoXwIwd+YMzmzTlu0797Ai5yv+3623Mn78ePr3D25mtXLlyvxtdO/enZdffrn8OyeRlcjVMkuI347+92KWuZfg5s8iEup47nl8tGZ1/nCdunVp9YM2rH7vP5zd6RwWvPQ8l/a7iq1bgvs9b9q0iebND95GNjk5ucrLLEeuw6JvGZHKNm7cOLKyssq07LdhTbyws85O5r8n3p/QOvbv38+yNxcx4OrrCoy/7MqBzH/xORqdcgo1atak8alN88N9/Pjx9OzZkx/+8IdceumljBgxghNPPBGAxYsXk5KSkr+eZ599ljPPPLP0OyeRpe4HRCrRnt27+HHv7lzYsRVff/UlXXpcVGB617SLWbr4Df4x71l6XzGgwLQRI0awZs0aBg8eTEZGBl26dGHPnuA2st27dycrKyv/oWCXwlRzl6PCH//4xzIvW9o291h16tZj7oLF7Pjma24ZPoQ5Mx7n2pGj86fXPuYY2iWnMHP6NJ5b+Db/+uf8Ass3a9aMkSNHMnLkSDp06MCqVasKb0IkLtXcRapA/QYn8OtJDzDzf/4/e/fuLTDthlE3M+62iZzY8KQC4+fPn58/7+bNm8nNzeW004r7cbjIQQp3kSrSrkNH2rTvwPwXny0w/gdntePKwUMPmf/VV1+lQ4cOdOrUid69e/OHP/yBJk2aAAfb3PMezzzzTJXsgxw5Dot7qKamprpu1iEVbc2aNbRr167c6yn1pZBfbACg9knlq2V3bH5iuZaXaIn3ejaz5e6eGm9+1dxFRCJI4S4iEkEKdxGRCFK4i4hEkMJdRCSCFO4iIhGkX6jKUSNpwisVur4Xx3at0PVVlbS0NKZMmUJqatwr6A6RkZHBlClTDumFMiMjg/79+9OqVSt27dpFv379mDIluKVyeno6I0aM4LXXXqNXr14APP/88wwcOJCnn36aQYMG8fLLL3PHHXdw4MAB9u7dy89//nNGjx7NxIkTeeyxx4i9iU9GRkZ+vzrxDB8+nH79+jFo0CB+8pOf8Itf/IL27dvHnTc9PZ1LL72UZs2axZ1+55130qNHDy6++GKSkpLIzMykUaNGCR2r7Oxs3nrrLa655hoAMjMzmTlzJo888khCy1ckhbtIFXB33D3//qpRkdf18K5duzjnnHMYMGAAXbsGH3rJycnMnj07P9znzJlDp06dgODm5aNGjWLZsmU0b96cPXv2kJ2dnb/e8ePH86tf/apMZXr88ceLnZ6enk6HDh3ihvv+/fuZNGlSmbYLQbg/+eST+eGempqa8IdoRYvWK03kMLLh88+46qLzuff2X3J1nwvZvDGH9L88wjV9ezLokq48+mDQo+R3333L2GE/ZvCl3RjY6wLmv/gcAJMmTeK8886jQ4cOjBo1irwfHKalpTF+/Hh69OhBu3bteOeddxg4cCCtW7fmt7/9LRCETNu2bRk2bBgdO3Zk0KBBfPfdd4eU8dVXX+WCCy7g3HPPZfDgwezcuRMIuj5o27Yt3bp147nnnitxX+vVq0dKSgobNmzIH9e9e3eWLVvG3r172blzJ+vWrcvvyXLHjh3s27ePk08+GYA6deqU6obi7s7YsWNp3749ffv2ZevWrfnT0tLSyMzMZP/+/QwfPpwOHTqQnJzM1KlTeeaZZ8jMzOTaa68lJSWFXbt2kZSUxKRJk+jWrRtPP/00w4cPL/CL3z/84Q907tyZzp07s27dOoBD5jn++OMBmDBhQv6vh6dOnUpGRgb9+vUD4IsvvuCqq66iY8eOdOnShRUrVgAwceJERo4cSVpaGq1ataqwWr7CXaQSZX/8EVcMGsLc+YvI/ngdn63/hFkvL2TugsW8vzKL5Uvf5K2MhTQ+tSlPv7qE5xa+Tde0oKY7duxY3nnnHVatWsWuXbsKNIscc8wxLFq0iDFjxtC/f3+mTZvGqlWrSE9PJzc3Fwju5DRq1ChWrFhBgwYNePTRRwuUbfv27dxzzz289tprvPvuu6SmpvLQQw+xe/dubrrpJl566SUWL17M5s2bS9zPL7/8ko8++ogePXrkjzMzLr74YhYsWMC8efO48sor86eddNJJXHnllZxxxhkMHTqUWbNmceDAgfzpU6dOze9a4aKLCvakCUETzwcffMDKlSt57LHHeOuttw6ZJysriw0bNrBq1SpWrlzJiBEjGDRoEKmpqcyaNYusrCzq1asHQN26dVmyZAlDhgw5ZD0NGjRg2bJljB07lnHjxhV7HCZPnpzfY+f48eMLTLvrrrs455xzWLFiBffddx833HBD/rS1a9eyYMECli1bxu9+97tD+h8qC4W7SCVq2rwFHc89D4C3F73B24te5+rLejCkz4Vkr/uIT7M/4Qdt27N0SQZT77uLd//9FvUbnADAG2+8wfnnn09ycjKvv/46q1cfvNFHXlAmJydz9tln07RpU+rUqUOrVq34/PPPAWjRokV+E8l1113HkiVLCpRt6dKlvP/++3Tt2pWUlBRmzJjBp59+ytq1a2nZsiWtW7fGzLjuuoJ90MdavHgxHTt2pEmTJvTr1y+/75s8Q4YMYc6cOcyZM4ehQwv2n/P444+zcOFCOnfuzJQpUxg5cmT+tPHjx+d3Z/zGG28cst1FixYxdOhQatasSbNmzejZs+ch87Rq1YpPPvmEW265hfnz59OgQYMi9+Pqq68uclpeuYcOHcrbb79d5HwlWbJkCddffz0APXv2JDc3l6+//hqAvn37UqdOHRo1asQpp5zCli1byrydPGpzF6lE9eodm//c3Rl583gGXzfikPnmvJLB4jde5eEHJnFBj56MGHMrP/vZz8jMzKRFixZMnDiR3bt3589fp04dAGrUqJH/PG94377g5iLB7Y8PKjzs7lxyySXMnj27wPisrKxD5i1KXpv7hx9+SLdu3RgwYECBm4h07tyZVatWUa9ePdq0aXPI8snJySQnJ3P99dfTsmVL0tPTE9puvP0prGHDhrz33nssWLCAadOmMXfuXJ544om48x533HEJbSfvea1atfLPNNyd77//vsTyxuvHK299sf/DmjVr5v8Py0M1d5Eq8sMLe/LCU7P47tugXXvLpo3kbt/G1s2bqFuvHv0GXs2wUbewduV7+TflaNSoETt37ixTr4+fffZZfk1z9uzZdOvWrcD0Ll268Oabb+a3I3/33Xd8+OGHtG3blvXr1/Pxxx/nL1uSNm3acNttt/HAAw8cMu3+++/nvvvuKzBu586dZGRk5A9nZWVxxhlnJLxvPXr0YM6cOezfv59NmzbFrd1v376dAwcO8KMf/Yi7776bd999F4D69euzY8eOhLf11FNP5f+94IILAEhKSmL58uUAzJs3L78Zpbh19+jRg1mzZgHB1T+NGjUq9myivFRzl6NG9uS+ZVquPDfriPXDC3uyft2HXN//UgCOPe547nv4f/gs+xOm3nsnNWrUoFat2vzmvgdpcMIJ3HTTTSQnJ5OUlMR5551X6u21a9eOGTNmMHr0aFq3bs1Pf/rTAtMbN25Meno6Q4cOzf8wueeee2jTpg3Tp0+nb9++NGrUiG7duiV0k5AxY8YwZcoU1q9fX2B8nz59DpnX3fn973/P6NGjqVevHscdd1yBWvvUqVP529/+lj/8wgsvkJSUlD88YMAAXn/9dZKTk2nTpg0XXnjhIdvYsGEDI0aMyK9h339/8AX28OHDGTNmDPXq1UuomWXPnj2cf/75HDhwIP+D7qabbqJ///507tyZXr165df8O3bsSK1atejUqRPDhw/nnHPOyV/PxIkTGTFiBB07duTYY49lxowZJW67PNTlr0TW0dzlb3Z2Nv369dOdmyJEXf6KiIjCXSSKkpKSVGs/yincJdIOh2ZHkfIqy+tY4S6RVbduXXJzcxXwckRzd3Jzc6lbt26pltPVMhJZzZs3Jycnh23btpVrPVu+3FWq+Q98+yUANXbuKdd21+yoV67lJTrq1q1L8+bNS7WMwl0iq3bt2rRs2bLc6+lTyt4kNz85AYAm10wu13bLeummCKhZRkQkkhTuIiIRpHAXEYkghbuISAQp3EVEIkjhLiISQSWGu5m1MLM3zGyNma02s5+H408ys3+a2Ufh34Yxy9xmZuvM7AMz612ZOyAiIodKpOa+D/ilu7cDugA3m1l7YAKw0N1bAwvDYcJpQ4CzgcuAR82sZmUUXkRE4isx3N19k7u/Gz7fAawBTgP6A3kdEs8Argqf9wfmuPsed18PrAM6V3C5RUSkGKX6haqZJQHnAP8GTnX3TRB8AJjZKeFspwFLYxbLCccVXtcoYBTA6aefXuqCi0RdUil/GVuR9OvYI1/CX6ia2fHAs8A4d/+muFnjjDuk5yZ3n+7uqe6e2rhx40SLISIiCUgo3M2sNkGwz3L358LRW8ysaTi9KbA1HJ8DtIhZvDmwsWKKKyIiiUjkahkD/gqscfeHYia9CAwLnw8D5sWMH2JmdcysJdAaWFZxRRYRkZIk0ubeFbgeWGlmWeG424HJwFwzuxH4DBgM4O6rzWwu8D7BlTY3u/v+ii64iIgUrcRwd/clxG9HB+hVxDL3AveWo1wiIlIO+oWqiEgEKdxFRCJI4S4iEkEKdxGRCFK4i4hEkMJdRCSCStW3jEh1qc5+VkSORKq5i4hEkMJdRCSCFO4iIhGkcBcRiSCFu4hIBCncRUQiSOEuIhJBCncRkQhSuIuIRJDCXUQkghTuIiIRpHAXEYkghbuISAQp3EVEIkjhLiISQQp3EZEIUriLiESQwl1EJIIU7iIiEaRwFxGJIIW7iEgEKdxFRCJI4S4iEkEKdxGRCKpV3QWQI0vShFequwgikgDV3EVEIqjEcDezJ8xsq5mtihk30cw2mFlW+Lg8ZtptZrbOzD4ws96VVXARESlaIjX3dOCyOOOnuntK+Pg7gJm1B4YAZ4fLPGpmNSuqsCIikpgSw93dFwFfJLi+/sAcd9/j7uuBdUDncpRPRETKoDxt7mPNbEXYbNMwHHca8HnMPDnhOBERqUJlDfc/A2cCKcAm4MFwvMWZ1+OtwMxGmVmmmWVu27atjMUQEZF4yhTu7r7F3fe7+wHgMQ42veQALWJmbQ5sLGId09091d1TGzduXJZiiIhIEcoU7mbWNGZwAJB3Jc2LwBAzq2NmLYHWwLLyFVFEREqrxB8xmdlsIA1oZGY5wF1AmpmlEDS5ZAOjAdx9tZnNBd4H9gE3u/v+Sim5iIgUqcRwd/ehcUb/tZj57wXuLU+hRESkfPQLVRGRCFK4i4hEkMJdRCSCFO4iIhGkcBcRiSCFu4hIBCncRUQiSOEuIhJBCncRkQhSuIuIRJDCXUQkghTuIiIRpHAXEYkghbuISAQp3EVEIkjhLiISQQp3EZEIUriLiESQwl1EJIIU7iIiEaRwFxGJIIW7iEgEKdxFRCJI4S4iEkEKdxGRCFK4i4hEkMJdRCSCFO4iIhGkcBcRiSCFu4hIBCncRUQiSOEuIhJBCncRkQgqMdzN7Akz22pmq2LGnWRm/zSzj8K/DWOm3WZm68zsAzPrXVkFFxGRoiVSc08HLis0bgKw0N1bAwvDYcysPTAEODtc5lEzq1lhpRURkYSUGO7uvgj4otDo/sCM8PkM4KqY8XPcfY+7rwfWAZ0rpqgiIpKosra5n+rumwDCv6eE408DPo+ZLyccJyIiVaiiv1C1OOM87oxmo8ws08wyt23bVsHFEBE5upU13LeYWVOA8O/WcHwO0CJmvubAxngrcPfp7p7q7qmNGzcuYzFERCSesob7i8Cw8PkwYF7M+CFmVsfMWgKtgWXlK6KIiJRWrZJmMLPZQBrQyMxygLuAycBcM7sR+AwYDODuq81sLvA+sA+42d33V1LZRUSkCCWGu7sPLWJSryLmvxe4tzyFEhGR8ikx3EXk6JM04ZVq2W725L7Vst0oUvcDIiIRpHAXEYkghbuISAQp3EVEIkjhLiISQQp3EZEIUriLiESQwl1EJIIU7iIiEaRwFxGJIIW7iEgEKdxFRCJI4S4iEkEKdxGRCFK4i4hEkMJdRCSCFO4iIhGkcBcRiSCFu4hIBCncRUQiSOEuIhJBCncRkQhSuIuIRJDCXUQkghTuIiIRpHAXEYkghbuISAQp3EVEIkjhLiISQQp3EZEIUriLiESQwl1EJIIU7iIiEVSrPAubWTawA9gP7HP3VDM7CXgKSAKygR+7+5flK6bESprwSnUXQUQOcxVRc7/I3VPcPTUcngAsdPfWwMJwWEREqlBlNMv0B2aEz2cAV1XCNkREpBjlDXcHXjWz5WY2Khx3qrtvAgj/nhJvQTMbZWaZZpa5bdu2chZDRERilavNHejq7hvN7BTgn2a2NtEF3X06MB0gNTXVy1kOERGJUa6au7tvDP9uBZ4HOgNbzKwpQPh3a3kLKSIipVPmcDez48ysft5z4FJgFfAiMCycbRgwr7yFFBGR0ilPs8ypwPNmlreeJ919vpm9A8w1sxuBz4DB5S+miIiURpnD3d0/ATrFGZ8L9CpPoUREpHz0C1URkQhSuIuIRJDCXUQkghTuIiIRpHAXEYkghbuISAQp3EVEIkjhLiISQQp3EZEIUriLiESQwl1EJIIU7iIiEaRwFxGJIIW7iEgEKdxFRCJI4S4iEkHlvUG2iEiFSZrwSrVsN3ty32rZbmVSzV1EJIIU7iIiEaRwFxGJILW5l0N1tQ+KiJRENXcRkQhSzV1EjnrVeRZeWVfqqOYuIhJBCncRkQhSuIuIRJDCXUQkghTuIiIRpHAXEYkghbuISAQp3EVEIkjhLiISQZH4har6eBERKajSau5mdpmZfWBm68xsQmVtR0REDlUp4W5mNYFpQB+gPTDUzNpXxrZERORQlVVz7wysc/dP3P17YA7Qv5K2JSIihVRWm/tpwOcxwznA+bEzmNkoYFQ4uNPMPqikshwuGgHbq7sQh4mj4lh8+kC/RGc9Ko5Hgo66Y2EPFDu5pONxRlETKivcLc44LzDgPh2YXknbP+yYWaa7p1Z3OQ4HOhYF6XgcpGNRUHmOR2U1y+QALWKGmwMbK2lbIiJSSGWF+ztAazNraWbHAEOAFytpWyIiUkilNMu4+z4zGwssAGoCT7j76srY1hHkqGmCSoCORUE6HgfpWBRU5uNh7l7yXCIickRR9wMiIhGkcBcRiSCFewUqqcsFMzvBzF4ys/fMbLWZjaiOclaVBI5HQzN73sxWmNkyM+tQHeWsCmb2hJltNbNVRUw3M3skPFYrzOzcqi5jVUngWLQ1s7fNbI+Z/aqqy1fVEjge14aviRVm9paZdUpkvQr3CpJglws3A++7eycgDXgwvJoochI8HrcDWe7eEbgBeLhqS1ml0oHLipneB2gdPkYBf66CMlWXdIo/Fl8AtwJTqqQ01S+d4o/HeuDC8H1yNwl+yapwrziJdLngQH0zM+B4ghfxvqotZpVJ5Hi0BxYCuPtaIMnMTq3aYlYNd19E8P8uSn9gpgeWAieaWdOqKV3VKulYuPtWd38H2Ft1pao+CRyPt9z9y3BwKcHvhkqkcK848bpcOK3QPH8C2hH8oGsl8HN3P1A1xatyiRyP94CBAGbWmeCn1Am9cCMokeMlciPwj0RmVLhXnBK7XAB6A1lAMyAF+JOZNajcYlWbRI7HZKChmWUBtwD/IbpnMiVJ5HjJUczMLiII918nMn8kbtZxmEiky4URwGQPflywzszWA22BZVVTxCpV4vFw928IjglhU9X68HE0UpcdUiQz6wg8DvRx99xEllHNveIk0uXCZ0AvgLBt+SzgkyotZdUp8XiY2YkxXyj/BFgUBv7R6EXghvCqmS7A1+6+qboLJdXPzE4HngOud/cPE11ONfcKUlSXC2Y2Jpz+F4JvutPNbCXBafiv3T2S3ZsmeDzaATPNbD/wPsEpZySZ2WyCK6QamVkOcBdQG/KPxd+By4F1wHeEZzRRVNKxMLMmQCbQADhgZuOA9lH94E/gtXEncDLwaHCCy75EeopU9wMiIhGkZhkRkQhSuIuIRJDCXUQkghTuIiIRpHAXEYkghbuISAQp3EVEIuj/ADYTFxgMwf4FAAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAkgUlEQVR4nO3deXhU5d3/8feXRUAFRUEBQQMWBCQQfSJiWYygIoIiFCq4sViBVrTQ9vcUbVWKG7Yo1V+xfdB6BZ4iiCsuLVjRFFApBhtZBBUlatiJGyggy/f545yESZgkkx0On9d1zZU5+31OZj5zn3vO3MfcHRERiZYa1V0AERGpeAp3EZEIUriLiESQwl1EJIIU7iIiEaRwFxGJIIX7EcrMJprZ36q7HEcDMzvdzHaaWc0q2Fb+/7Wit2tmfzGzO8LnaWaWUxHrDdfX3cw+qKj1VRYzyzCzn1R3OaqCwj0BZpZtZrvCN9pmM0s3s+NjpqebmZvZlYWW+2M4fng4fIyZPWhmOeG61pvZ1CK2k/f4UyXsjz4YSsHdP3P34919/+G4XTMbbmZLEljfGHe/uyLKFr6ufxCz7sXuflZFrLs6mdn48D3+tZk9YWZ1qrtMZaVwT9wV7n48kAKcA9xWaPqHwLC8ATOrBQwGPo6Z5zYgFegM1AcuAv4Tbzsxj7EVuhdyVKqKs44jnZn1BiYAvYAkoBXwu+osU3ko3EvJ3TcDCwhCPtZLQFczaxgOXwasADbHzHMe8Ly7b/RAtrvPLEdx6prZU2a2w8zeNbNOeRPMrJmZPWtm28IzhFvD8ZcBtwNXh2cG75nZRWa2MmbZ18xsWczwEjO7qrj1htNqmNkEM/vYzHLNbK6ZnRROSwpre8PM7DMz225mvylqx8zsZDN7ycy+MbN3zOye2NqpmT1sZp+H05ebWfeYaelmdk/McIEmCDP7tZltCI/bB2bWKxzf2cwyw3VuMbOHCpW9Vjg8wszWhMt/YmajC2/LzH5pZlvNbJOZjShmP1ua2b/Cdf0TaBQzrfB2h4fb2xEe+2vNrB3wF+CC8P/5Vcwx+LOZ/d3MvgUuKnxcwvluD/8X2WZ2bcz4As0XsWcHZrYoHP1euM2r4xzjduE6vjKz1RZzVhuWY5qZvRLuy7/N7MxijtHTdrA2vcjMzk50XWZ2iZmtDZf9E2BFbYegcvZXd1/t7l8CdwPDi5n/sKZwLyUzaw70AdYVmrQbeBEYEg7fABQO7qXAL8zsZ2aWbGbFvdAS0R94GjgJeBJ4wcxqm1kNgg+b94DTCGoi48yst7vPB+4DngrPDDoBbwM/MLNGYZB0AJqbWX0zqwf8F7C4uPWG5bkVuAq4EGgGfAlMK1TmbsBZ4bJ3huEUzzTgW6AJwZtuWKHp7xB8wObt+9NmVrekA2ZmZwFjgfPcvT7QG8gOJz8MPOzuDYAzgblFrGYr0A9oAIwApprZuTHTmwAnEByjG4FpdvBDv7AngeUEoX53nP3MK/dxwCNAn7DcPwSy3H0NMAZ4O/x/nhiz2DXAvQRnifGabZqE2z0t3O708PgUy917hE87hdt8qlBZaxO8Tl4FTgFuAWYVWvdQglpxQ4L30r3FbPIfQOtwXe8CswpNj7suM2sEPAv8NtzPj4GuxWznbILXdp73gFPN7ORiljlsKdwT94KZ7QA+J3hz3xVnnpnADWZ2AkHAvVBo+v3AA8C1QCawwcwKv5lfCGs7eY+biinTcnd/xt33Ag8BdYEuBGcIjd19krt/7+6fAI9x8IOnAHffHZanB0Gz0QqCMOgaru8jd89NYL2jgd+4e4677wEmAoPyap6h37n7Lnd/j+DN04lCLGhC+BFwl7t/5+7vAzMKlflv7p7r7vvc/UGgDsGHRkn2h/O2N7Pa4dlTXtPZXsIPOXff6e5Lizher7j7x+HZ178IQqx7zCx7gUnuvtfd/w7sjFc2Mzud4Jje4e573H0RQSgW5QDQwczqufsmd19dwr7Oc/c33f1A+D+OJ2/b/wJeAX5cwjoT0QU4Hpgcvk5eB14mCOE8z7n7MnffRxDWKUWtzN2fcPcdMa+pTuF7rKR1XQ68H/Me+SMFz6QLOx74OmY473n94nb2cKVwT9xVYY0pDWhLzOlzHndfAjQmqCm87O67Ck3f7+7T3L0rcCJBDeOJQrXXq9z9xJjHY8WU6fOYdR8AcghqzGcAzWI/JAiaYk4tZl3/CvetR/g8g+AD6sJwmATWewbwfMy0NQRhGrvd2DfXdwRvqMIaA7Vi96/Qc8JmjzXh6fZXBDXlQ/4nhbn7OmAcQUhsNbM5ZtYsnHwj0AZYa0FTUL946zCzPma21My+CLd9eaFt54ZBU9J+NgO+dPdvY8Z9WkS5vwWuJqilbwqbIdoWv7cFj1kc8bbdrKiZS6EZ8Hn4moxd92kxw4m8DjCzmmY22YKmvm84eJYVe7yLWlczCr5HnOKPyU6Cs7E8ec93FLPMYUvhXkphDScdmFLELH8DfsmhTTKF17PL3acRNF20L2NxWuQ9CZtMmgMbCV7A6wt9SNR398vzNh9nXYXD/V8cGu4lrfdzgmaD2Ol13X1DKfdrG7Av3J94+9od+DVBLbNh2BTxNQfbU78Fjo1Ztknsyt39SXfvRvBh5ARnU7j7R+4+lOD0/wHgmbA5JJ8FV088S/D/PzXc9t8pvi23KJuAhoW2cXpRM7v7Ane/BGgKrCU4a4L4/8/ixueJt+2N4fNij2EJNgItwtdk7LpL+zqAoGmpP3AxwQd4Ujg+keO9iYKvG4sdjmM1Bc8kOwFbwrPWI47CvWz+CFxiZilxpj0CXAIsKjzBzMaFXzzVM7NaYZNMfQ69YiZR/2VmA8Nmj3HAHoJ2/WXANxZ8cVgvrP10MLPzwuW2AEmF3nxvETQddAaWhaf8ZwDnx+xLSev9C3CvmZ0R7m9jM+tf2p3y4NK/54CJZnZsWEO9IWaW+gThvw2oZWZ3UrDGlQVcbmYnmVmT8NgQluksM+sZhvRuYBfB2QVmdp2ZNQ5rnF+FixS+DPEYgmadbcA+M+sDXFrafQz381OC5rDfWXCZbDfginjzmtmpZnZlGMZ7CGqZeWXbQvAdyTFlKEbetrsTfI/wdDg+CxgYHv8fEJzVxNpCcDVJPP8m+HD47/A7oLRwv+aUoXz1CfY3l+DD5r5SLPsKcHbMe+RWiv+QmgncaGbtw+9IfktQkTsiKdzLwN23EbwQ7ogz7Qt3XxieAha2C3iQ4DRyO3Az8KOw7TrPS1bwOvfniynKPIJT9S+B64GBYTvvfoI3UwqwPtzW4wQ1Hzj4Bs41s3fDcn9L8GXVanf/Ppz+NvCpu28N5ylpvQ8TfKn8avj9xFKCD4eyGBuudzPwv8Bsgjc5BFcr/YPg8tNPCUI69nT7fwna87MJ2sNjv/CrA0wOy76ZoJZ+ezjtMmC1me0M92VI4bZqd99BEBJzCY77NeE+l9U1BMfoC4LvcYo646tBcEa4MZz3QuBn4bTXCWqdm81seym2vZlgHzYStFWPcfe14bSpwPcEIT6DQ7/EnAjMCJvgCrTTh6+fKwkuPNgOPArcELPu0phJ8D/eALxP8JpKiLtvJ7gceTLBh0Nr4M1i5p8P/B54I9zmp8T/bu2IYPEzSOTwYmYPAE3cPe7VJCJSkGruclgys7Zm1tECnQmaBYo7ixGRGLVKnkWkWtQnaIppRnDp6YMEzVAikgA1y4iIRJCaZUREIuiwaJZp1KiRJyUlVXcxRCrEBx8EPd+eddYR30miHOaWL1++3d0bx5t2WIR7UlISmZmZ1V0MkQqRlpYGQEZGRrWWQ6LPzOL+ohnULCMiEkkKdxGRCCox3M2shZm9EXbStNrMfh6On2hBn9hZ4ePymGVuM7N1FvSV3bvotYuISGVIpM19H/BLd3/XzOoDyy24qQDAVHcv0IGWmbUn6AL2bIJrlF8zszZexbcoE9m7dy85OTns3l1Ub7eV4667gl+sr1mzpkq3K9FVt25dmjdvTu3atRNepsRwd/dNBL2r4e47zGwNBbvuLKw/MCfse3m9ma0j6Izq7YRLJVIBcnJyqF+/PklJSVi574uSuBo1ghNiXS0jFcHdyc3NJScnh5YtWya8XKna3M0sieD+of8OR401sxUW3Eg2704zp1GwE6cc4nwYmNkoC25plrlt27bSFEMkIbt37+bkk0+u0mAXqWhmxsknn1zqM9CEw93Mjifox3qcu38D/JngVmQpBDX7B/NmjbP4IT+Ddffp7p7q7qmNG8e9TFOk3BTsEgVleR0nFO4W3BPxWWCWuz8H4O5bwjsLHSC4aUDncPYcCnaIn3cDCRERqSKJXC1jwF+BNe7+UMz4pjGzDQBWhc9fBIaYWR0za0nQh/KyiiuyyJGjZs2apKSk0KFDB6644gq++uorALKzszEz7rjj4C0Btm/fTu3atRk7diwQ/NI1LS2NlJQU2rVrx6hRo4Dgx1EnnHACKSkp+Y/XXnutyvdNDm+JXC3TleBGECvNLCscdzswNLwTkRPcFGE0gLuvNrO5BB3r7wNu1pUyciRbkfNVqebfu2df/nJ16tZj5ssZAPx2/E+5474p3HTrr9iw6Ruan57E08/PY/DoXwIwd+YMzmzTlu0797Ai5yv+3623Mn78ePr3D25mtXLlyvxtdO/enZdffrn8OyeRlcjVMkuI347+92KWuZfg5s8iEup47nl8tGZ1/nCdunVp9YM2rH7vP5zd6RwWvPQ8l/a7iq1bgvs9b9q0iebND95GNjk5ucrLLEeuw6JvGZHKNm7cOLKyssq07LdhTbyws85O5r8n3p/QOvbv38+yNxcx4OrrCoy/7MqBzH/xORqdcgo1atak8alN88N9/Pjx9OzZkx/+8IdceumljBgxghNPPBGAxYsXk5KSkr+eZ599ljPPPLP0OyeRpe4HRCrRnt27+HHv7lzYsRVff/UlXXpcVGB617SLWbr4Df4x71l6XzGgwLQRI0awZs0aBg8eTEZGBl26dGHPnuA2st27dycrKyv/oWCXwlRzl6PCH//4xzIvW9o291h16tZj7oLF7Pjma24ZPoQ5Mx7n2pGj86fXPuYY2iWnMHP6NJ5b+Db/+uf8Ass3a9aMkSNHMnLkSDp06MCqVasKb0IkLtXcRapA/QYn8OtJDzDzf/4/e/fuLTDthlE3M+62iZzY8KQC4+fPn58/7+bNm8nNzeW004r7cbjIQQp3kSrSrkNH2rTvwPwXny0w/gdntePKwUMPmf/VV1+lQ4cOdOrUid69e/OHP/yBJk2aAAfb3PMezzzzTJXsgxw5Dot7qKamprpu1iEVbc2aNbRr167c6yn1pZBfbACg9knlq2V3bH5iuZaXaIn3ejaz5e6eGm9+1dxFRCJI4S4iEkEKdxGRCFK4i4hEkMJdRCSCFO4iIhGkX6jKUSNpwisVur4Xx3at0PVVlbS0NKZMmUJqatwr6A6RkZHBlClTDumFMiMjg/79+9OqVSt27dpFv379mDIluKVyeno6I0aM4LXXXqNXr14APP/88wwcOJCnn36aQYMG8fLLL3PHHXdw4MAB9u7dy89//nNGjx7NxIkTeeyxx4i9iU9GRkZ+vzrxDB8+nH79+jFo0CB+8pOf8Itf/IL27dvHnTc9PZ1LL72UZs2axZ1+55130qNHDy6++GKSkpLIzMykUaNGCR2r7Oxs3nrrLa655hoAMjMzmTlzJo888khCy1ckhbtIFXB33D3//qpRkdf18K5duzjnnHMYMGAAXbsGH3rJycnMnj07P9znzJlDp06dgODm5aNGjWLZsmU0b96cPXv2kJ2dnb/e8ePH86tf/apMZXr88ceLnZ6enk6HDh3ihvv+/fuZNGlSmbYLQbg/+eST+eGempqa8IdoRYvWK03kMLLh88+46qLzuff2X3J1nwvZvDGH9L88wjV9ezLokq48+mDQo+R3333L2GE/ZvCl3RjY6wLmv/gcAJMmTeK8886jQ4cOjBo1irwfHKalpTF+/Hh69OhBu3bteOeddxg4cCCtW7fmt7/9LRCETNu2bRk2bBgdO3Zk0KBBfPfdd4eU8dVXX+WCCy7g3HPPZfDgwezcuRMIuj5o27Yt3bp147nnnitxX+vVq0dKSgobNmzIH9e9e3eWLVvG3r172blzJ+vWrcvvyXLHjh3s27ePk08+GYA6deqU6obi7s7YsWNp3749ffv2ZevWrfnT0tLSyMzMZP/+/QwfPpwOHTqQnJzM1KlTeeaZZ8jMzOTaa68lJSWFXbt2kZSUxKRJk+jWrRtPP/00w4cPL/CL3z/84Q907tyZzp07s27dOoBD5jn++OMBmDBhQv6vh6dOnUpGRgb9+vUD4IsvvuCqq66iY8eOdOnShRUrVgAwceJERo4cSVpaGq1ataqwWr7CXaQSZX/8EVcMGsLc+YvI/ngdn63/hFkvL2TugsW8vzKL5Uvf5K2MhTQ+tSlPv7qE5xa+Tde0oKY7duxY3nnnHVatWsWuXbsKNIscc8wxLFq0iDFjxtC/f3+mTZvGqlWrSE9PJzc3Fwju5DRq1ChWrFhBgwYNePTRRwuUbfv27dxzzz289tprvPvuu6SmpvLQQw+xe/dubrrpJl566SUWL17M5s2bS9zPL7/8ko8++ogePXrkjzMzLr74YhYsWMC8efO48sor86eddNJJXHnllZxxxhkMHTqUWbNmceDAgfzpU6dOze9a4aKLCvakCUETzwcffMDKlSt57LHHeOuttw6ZJysriw0bNrBq1SpWrlzJiBEjGDRoEKmpqcyaNYusrCzq1asHQN26dVmyZAlDhgw5ZD0NGjRg2bJljB07lnHjxhV7HCZPnpzfY+f48eMLTLvrrrs455xzWLFiBffddx833HBD/rS1a9eyYMECli1bxu9+97tD+h8qC4W7SCVq2rwFHc89D4C3F73B24te5+rLejCkz4Vkr/uIT7M/4Qdt27N0SQZT77uLd//9FvUbnADAG2+8wfnnn09ycjKvv/46q1cfvNFHXlAmJydz9tln07RpU+rUqUOrVq34/PPPAWjRokV+E8l1113HkiVLCpRt6dKlvP/++3Tt2pWUlBRmzJjBp59+ytq1a2nZsiWtW7fGzLjuuoJ90MdavHgxHTt2pEmTJvTr1y+/75s8Q4YMYc6cOcyZM4ehQwv2n/P444+zcOFCOnfuzJQpUxg5cmT+tPHjx+d3Z/zGG28cst1FixYxdOhQatasSbNmzejZs+ch87Rq1YpPPvmEW265hfnz59OgQYMi9+Pqq68uclpeuYcOHcrbb79d5HwlWbJkCddffz0APXv2JDc3l6+//hqAvn37UqdOHRo1asQpp5zCli1byrydPGpzF6lE9eodm//c3Rl583gGXzfikPnmvJLB4jde5eEHJnFBj56MGHMrP/vZz8jMzKRFixZMnDiR3bt3589fp04dAGrUqJH/PG94377g5iLB7Y8PKjzs7lxyySXMnj27wPisrKxD5i1KXpv7hx9+SLdu3RgwYECBm4h07tyZVatWUa9ePdq0aXPI8snJySQnJ3P99dfTsmVL0tPTE9puvP0prGHDhrz33nssWLCAadOmMXfuXJ544om48x533HEJbSfvea1atfLPNNyd77//vsTyxuvHK299sf/DmjVr5v8Py0M1d5Eq8sMLe/LCU7P47tugXXvLpo3kbt/G1s2bqFuvHv0GXs2wUbewduV7+TflaNSoETt37ixTr4+fffZZfk1z9uzZdOvWrcD0Ll268Oabb+a3I3/33Xd8+OGHtG3blvXr1/Pxxx/nL1uSNm3acNttt/HAAw8cMu3+++/nvvvuKzBu586dZGRk5A9nZWVxxhlnJLxvPXr0YM6cOezfv59NmzbFrd1v376dAwcO8KMf/Yi7776bd999F4D69euzY8eOhLf11FNP5f+94IILAEhKSmL58uUAzJs3L78Zpbh19+jRg1mzZgHB1T+NGjUq9myivFRzl6NG9uS+ZVquPDfriPXDC3uyft2HXN//UgCOPe547nv4f/gs+xOm3nsnNWrUoFat2vzmvgdpcMIJ3HTTTSQnJ5OUlMR5551X6u21a9eOGTNmMHr0aFq3bs1Pf/rTAtMbN25Meno6Q4cOzf8wueeee2jTpg3Tp0+nb9++NGrUiG7duiV0k5AxY8YwZcoU1q9fX2B8nz59DpnX3fn973/P6NGjqVevHscdd1yBWvvUqVP529/+lj/8wgsvkJSUlD88YMAAXn/9dZKTk2nTpg0XXnjhIdvYsGEDI0aMyK9h339/8AX28OHDGTNmDPXq1UuomWXPnj2cf/75HDhwIP+D7qabbqJ///507tyZXr165df8O3bsSK1atejUqRPDhw/nnHPOyV/PxIkTGTFiBB07duTYY49lxowZJW67PNTlr0TW0dzlb3Z2Nv369dOdmyJEXf6KiIjCXSSKkpKSVGs/yincJdIOh2ZHkfIqy+tY4S6RVbduXXJzcxXwckRzd3Jzc6lbt26pltPVMhJZzZs3Jycnh23btpVrPVu+3FWq+Q98+yUANXbuKdd21+yoV67lJTrq1q1L8+bNS7WMwl0iq3bt2rRs2bLc6+lTyt4kNz85AYAm10wu13bLeummCKhZRkQkkhTuIiIRpHAXEYkghbuISAQp3EVEIkjhLiISQSWGu5m1MLM3zGyNma02s5+H408ys3+a2Ufh34Yxy9xmZuvM7AMz612ZOyAiIodKpOa+D/ilu7cDugA3m1l7YAKw0N1bAwvDYcJpQ4CzgcuAR82sZmUUXkRE4isx3N19k7u/Gz7fAawBTgP6A3kdEs8Argqf9wfmuPsed18PrAM6V3C5RUSkGKX6haqZJQHnAP8GTnX3TRB8AJjZKeFspwFLYxbLCccVXtcoYBTA6aefXuqCi0RdUil/GVuR9OvYI1/CX6ia2fHAs8A4d/+muFnjjDuk5yZ3n+7uqe6e2rhx40SLISIiCUgo3M2sNkGwz3L358LRW8ysaTi9KbA1HJ8DtIhZvDmwsWKKKyIiiUjkahkD/gqscfeHYia9CAwLnw8D5sWMH2JmdcysJdAaWFZxRRYRkZIk0ubeFbgeWGlmWeG424HJwFwzuxH4DBgM4O6rzWwu8D7BlTY3u/v+ii64iIgUrcRwd/clxG9HB+hVxDL3AveWo1wiIlIO+oWqiEgEKdxFRCJI4S4iEkEKdxGRCFK4i4hEkMJdRCSCStW3jEh1qc5+VkSORKq5i4hEkMJdRCSCFO4iIhGkcBcRiSCFu4hIBCncRUQiSOEuIhJBCncRkQhSuIuIRJDCXUQkghTuIiIRpHAXEYkghbuISAQp3EVEIkjhLiISQQp3EZEIUriLiESQwl1EJIIU7iIiEaRwFxGJIIW7iEgEKdxFRCJI4S4iEkEKdxGRCKpV3QWQI0vShFequwgikgDV3EVEIqjEcDezJ8xsq5mtihk30cw2mFlW+Lg8ZtptZrbOzD4ws96VVXARESlaIjX3dOCyOOOnuntK+Pg7gJm1B4YAZ4fLPGpmNSuqsCIikpgSw93dFwFfJLi+/sAcd9/j7uuBdUDncpRPRETKoDxt7mPNbEXYbNMwHHca8HnMPDnhOBERqUJlDfc/A2cCKcAm4MFwvMWZ1+OtwMxGmVmmmWVu27atjMUQEZF4yhTu7r7F3fe7+wHgMQ42veQALWJmbQ5sLGId09091d1TGzduXJZiiIhIEcoU7mbWNGZwAJB3Jc2LwBAzq2NmLYHWwLLyFVFEREqrxB8xmdlsIA1oZGY5wF1AmpmlEDS5ZAOjAdx9tZnNBd4H9gE3u/v+Sim5iIgUqcRwd/ehcUb/tZj57wXuLU+hRESkfPQLVRGRCFK4i4hEkMJdRCSCFO4iIhGkcBcRiSCFu4hIBCncRUQiSOEuIhJBCncRkQhSuIuIRJDCXUQkghTuIiIRpHAXEYkghbuISAQp3EVEIkjhLiISQQp3EZEIUriLiESQwl1EJIIU7iIiEaRwFxGJIIW7iEgEKdxFRCJI4S4iEkEKdxGRCFK4i4hEkMJdRCSCFO4iIhGkcBcRiSCFu4hIBCncRUQiSOEuIhJBCncRkQgqMdzN7Akz22pmq2LGnWRm/zSzj8K/DWOm3WZm68zsAzPrXVkFFxGRoiVSc08HLis0bgKw0N1bAwvDYcysPTAEODtc5lEzq1lhpRURkYSUGO7uvgj4otDo/sCM8PkM4KqY8XPcfY+7rwfWAZ0rpqgiIpKosra5n+rumwDCv6eE408DPo+ZLyccJyIiVaiiv1C1OOM87oxmo8ws08wyt23bVsHFEBE5upU13LeYWVOA8O/WcHwO0CJmvubAxngrcPfp7p7q7qmNGzcuYzFERCSesob7i8Cw8PkwYF7M+CFmVsfMWgKtgWXlK6KIiJRWrZJmMLPZQBrQyMxygLuAycBcM7sR+AwYDODuq81sLvA+sA+42d33V1LZRUSkCCWGu7sPLWJSryLmvxe4tzyFEhGR8ikx3EXk6JM04ZVq2W725L7Vst0oUvcDIiIRpHAXEYkghbuISAQp3EVEIkjhLiISQQp3EZEIUriLiESQwl1EJIIU7iIiEaRwFxGJIIW7iEgEKdxFRCJI4S4iEkEKdxGRCFK4i4hEkMJdRCSCFO4iIhGkcBcRiSCFu4hIBCncRUQiSOEuIhJBCncRkQhSuIuIRJDCXUQkghTuIiIRpHAXEYkghbuISAQp3EVEIkjhLiISQQp3EZEIUriLiESQwl1EJIIU7iIiEVSrPAubWTawA9gP7HP3VDM7CXgKSAKygR+7+5flK6bESprwSnUXQUQOcxVRc7/I3VPcPTUcngAsdPfWwMJwWEREqlBlNMv0B2aEz2cAV1XCNkREpBjlDXcHXjWz5WY2Khx3qrtvAgj/nhJvQTMbZWaZZpa5bdu2chZDRERilavNHejq7hvN7BTgn2a2NtEF3X06MB0gNTXVy1kOERGJUa6au7tvDP9uBZ4HOgNbzKwpQPh3a3kLKSIipVPmcDez48ysft5z4FJgFfAiMCycbRgwr7yFFBGR0ilPs8ypwPNmlreeJ919vpm9A8w1sxuBz4DB5S+miIiURpnD3d0/ATrFGZ8L9CpPoUREpHz0C1URkQhSuIuIRJDCXUQkghTuIiIRpHAXEYkghbuISAQp3EVEIkjhLiISQQp3EZEIUriLiESQwl1EJIIU7iIiEaRwFxGJIIW7iEgEKdxFRCJI4S4iEkHlvUG2iEiFSZrwSrVsN3ty32rZbmVSzV1EJIIU7iIiEaRwFxGJILW5l0N1tQ+KiJRENXcRkQhSzV1EjnrVeRZeWVfqqOYuIhJBCncRkQhSuIuIRJDCXUQkghTuIiIRpHAXEYkghbuISAQp3EVEIkjhLiISQZH4har6eBERKajSau5mdpmZfWBm68xsQmVtR0REDlUp4W5mNYFpQB+gPTDUzNpXxrZERORQlVVz7wysc/dP3P17YA7Qv5K2JSIihVRWm/tpwOcxwznA+bEzmNkoYFQ4uNPMPqikshwuGgHbq7sQh4mj4lh8+kC/RGc9Ko5Hgo66Y2EPFDu5pONxRlETKivcLc44LzDgPh2YXknbP+yYWaa7p1Z3OQ4HOhYF6XgcpGNRUHmOR2U1y+QALWKGmwMbK2lbIiJSSGWF+ztAazNraWbHAEOAFytpWyIiUkilNMu4+z4zGwssAGoCT7j76srY1hHkqGmCSoCORUE6HgfpWBRU5uNh7l7yXCIickRR9wMiIhGkcBcRiSCFewUqqcsFMzvBzF4ys/fMbLWZjaiOclaVBI5HQzN73sxWmNkyM+tQHeWsCmb2hJltNbNVRUw3M3skPFYrzOzcqi5jVUngWLQ1s7fNbI+Z/aqqy1fVEjge14aviRVm9paZdUpkvQr3CpJglws3A++7eycgDXgwvJoochI8HrcDWe7eEbgBeLhqS1ml0oHLipneB2gdPkYBf66CMlWXdIo/Fl8AtwJTqqQ01S+d4o/HeuDC8H1yNwl+yapwrziJdLngQH0zM+B4ghfxvqotZpVJ5Hi0BxYCuPtaIMnMTq3aYlYNd19E8P8uSn9gpgeWAieaWdOqKV3VKulYuPtWd38H2Ft1pao+CRyPt9z9y3BwKcHvhkqkcK848bpcOK3QPH8C2hH8oGsl8HN3P1A1xatyiRyP94CBAGbWmeCn1Am9cCMokeMlciPwj0RmVLhXnBK7XAB6A1lAMyAF+JOZNajcYlWbRI7HZKChmWUBtwD/IbpnMiVJ5HjJUczMLiII918nMn8kbtZxmEiky4URwGQPflywzszWA22BZVVTxCpV4vFw928IjglhU9X68HE0UpcdUiQz6wg8DvRx99xEllHNveIk0uXCZ0AvgLBt+SzgkyotZdUp8XiY2YkxXyj/BFgUBv7R6EXghvCqmS7A1+6+qboLJdXPzE4HngOud/cPE11ONfcKUlSXC2Y2Jpz+F4JvutPNbCXBafiv3T2S3ZsmeDzaATPNbD/wPsEpZySZ2WyCK6QamVkOcBdQG/KPxd+By4F1wHeEZzRRVNKxMLMmQCbQADhgZuOA9lH94E/gtXEncDLwaHCCy75EeopU9wMiIhGkZhkRkQhSuIuIRJDCXUQkghTuIiIRpHAXEYkghbuISAQp3EVEIuj/ADYTFxgMwf4FAAAAAElFTkSuQmCC\n", "text/plain": [ "
    " ] @@ -2149,7 +1769,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.12" + "version": "3.8.6" } }, "nbformat": 4, diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index 86248386..9ee207f9 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -478,6 +478,23 @@ def threshold_brier_score( return res.mean(dim, keep_attrs=keep_attrs) +def _assign_rps_category_bounds(res, edges, name, bin_dim="category_edge"): + """Add category_edge coord to rps return. + Additionally adds left-most -np.inf category and right-most +np.inf category.""" + if edges[bin_dim].size >= 2: + res = res.assign_coords( + { + f"{name}_category_edge": ", ".join( + _get_category_bounds(edges[bin_dim].values) + ) + } + ) + res[ + f"{name}_category_edge" + ] = f"[-np.inf, {edges[bin_dim].isel({bin_dim:0}).values}), {str(res[f'{name}_category_edge'].values)[:-1]}), [{edges[bin_dim].isel({bin_dim:-1}).values}, np.inf]" + return res + + def rps( observations, forecasts, @@ -510,24 +527,35 @@ def rps( The forecast of the event with dimension specified by ``member_dim``. Further requirements are specified based on ``category_edges``. category_edges : array_like, xr.Dataset, xr.DataArray, None - Edges (left-edge inclusive) of the bins used to calculate the cumulative density function (cdf). Note that here the bins have to include the full range of observations and forecasts data. Effectively, negative infinity is appended to the left side of category_edges, and positive infinity is appended to the right side. Thus, N category edges produces N+1 bins. For example, specifying category_edges = [0,1] will compute the cdfs for bins [-inf, 0), [-inf, 1) and [-inf, inf). Note that the edges are right-edge exclusive. + Edges (left-edge inclusive) of the bins used to calculate the cumulative + density function (cdf). Note that here the bins have to include the full range + of observations and forecasts data. Effectively, negative infinity is appended + to the left side of category_edges, and positive infinity is appended to the + right side. Thus, N category edges produces N+1 bins. For example, specifying + category_edges = [0,1] will compute the cdfs for bins [-inf, 0), [-inf, 1) and + [-inf, inf). Note that the edges are right-edge exclusive. Forecasts, observations and category_edge are expected in absolute units or probabilities consistently. - ``category_edges`` decides how xs.rps interprets forecasts and observations. - - np.array (1d): will be internally converted and broadcasted to observations. Use this if you wish to use the same category edges for all elements of both forecasts and observations. + - np.array (1d): will be internally converted and broadcasted to observations. + Use this if you wish to use the same category edges for all elements of both + forecasts and observations. - xr.Dataset/xr.DataArray: edges of the categories provided as dimension ``category_edge`` with optional category labels as ``category_edge`` coordinate. Use xr.Dataset/xr.DataArray if edges - multi-dimensional and vary across dimensions. Use this if your category edges vary across dimensions of forecasts and observations, but are the same for both. + multi-dimensional and vary across dimensions. Use this if your category edges + vary across dimensions of forecasts and observations, but are the same for + both. - tuple of np.array/xr.Dataset/xr.DataArray: same as above, where the first item is taken as ``category_edges`` for observations and the second item - for ``category_edges`` for forecasts. Use this if your category edges vary across dimensions of forecasts and observations, and are different for each. + for ``category_edges`` for forecasts. Use this if your category edges vary + across dimensions of forecasts and observations, and are different for each. - None: expect than observations and forecasts are already CDFs containing - ``category_edge`` dimension. Use this if your category edges vary across dimensions of forecasts and observations, and are different for each. + ``category_edge`` dimension. Use this if your category edges vary across + dimensions of forecasts and observations, and are different for each. dim : str or list of str, optional Dimension over which to mean after computing ``rps``. This represents a mean @@ -560,7 +588,7 @@ def rps( ... coords=[('x', np.arange(3)), ... ('y', np.arange(3)), ... ('member', np.arange(3))]) - >>> category_edges = np.array([.0, .5, 1.]) + >>> category_edges = np.array([.33, .66]) >>> xs.rps(observations, forecasts, category_edges, dim='x') array([0.14814815, 0.7037037 , 1.51851852]) @@ -670,26 +698,8 @@ def rps( # add category_edge as str into coords if category_edges is not None: - res = res.assign_coords( - { - "forecasts_category_edge": ", ".join( - _get_category_bounds(forecasts_edges[bin_dim].values) - ) - } - ) - res = res.assign_coords( - { - "observations_category_edge": ", ".join( - _get_category_bounds(observations_edges[bin_dim].values) - ) - } - ) - res[ - "forecasts_category_edge" - ] = f"[-np.inf, {forecasts_edges[bin_dim].isel(category_edge=0).values}), {str(res['forecasts_category_edge'].values)[:-1]}), [{forecasts_edges[bin_dim].isel(category_edge=-1).values}, np.inf]" - res[ - "observations_category_edge" - ] = f"[-np.inf, {observations_edges[bin_dim].isel(category_edge=0).values}), {str(res['observations_category_edge'].values)[:-1]}), [{observations_edges[bin_dim].isel(category_edge=-1).values}, np.inf]" + res = _assign_rps_category_bounds(res, observations_edges, "observations") + res = _assign_rps_category_bounds(res, forecasts_edges, "forecasts") if weights is not None: res = res.weighted(weights) # combine many forecasts-observations pairs diff --git a/xskillscore/tests/test_probabilistic.py b/xskillscore/tests/test_probabilistic.py index 78cba8a7..8217bef6 100644 --- a/xskillscore/tests/test_probabilistic.py +++ b/xskillscore/tests/test_probabilistic.py @@ -744,12 +744,15 @@ def test_rps_keeps_masked(o, f_prob, fair_bool, category_edges): assert set(["lon", "lat"]) == set(actual.dims) assert actual.isel(lat=[0, 1]).isnull().all() assert actual.isel(lat=slice(2, None)).notnull().all() - print(actual.coords["forecasts_category_edge"].values) # test forecasts_category_edge no repeats assert ( "[-np.inf, 0.2), [0.2, 0.4), [0.4, 0.6), [0.6, 0.8), [0.8, np.inf]" in actual.coords["forecasts_category_edge"].values ) + # one more category internally used than category_edges provided + assert len(category_edges) + 1 == str( + actual.coords["forecasts_category_edge"].values + ).count("[") @pytest.mark.parametrize("fair_bool", [True, False], ids=["bool=fair", "fair=False"]) From 20610bb38c586b705ab8aa483caa13ab07b7b023 Mon Sep 17 00:00:00 2001 From: Aaron Spring Date: Wed, 10 Mar 2021 22:40:45 +0100 Subject: [PATCH 33/39] Update CHANGELOG.rst --- CHANGELOG.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1912e792..1facfd6a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -23,7 +23,9 @@ Features - Added receiver operating characteristic (ROC) :py:func:`~xskillscore.roc`. (:issue:`114`, :issue:`256`, :pr:`236`, :pr:`259`) `Aaron Spring`_ - Added many options for ``category_edges`` in :py:func:`~xskillscore.rps`, which - allows multi-dimensional edges. (:issue:`275`, :pr:`277`) `Aaron Spring`_ + allows multi-dimensional edges. :py:func:`~xskillscore.rps` now + requires dimension ``member_dim`` in forecasts. (:issue:`275`, :pr:`277`) + `Aaron Spring`_ Breaking changes ~~~~~~~~~~~~~~~~ From 1b3e36b76716a9d010ca0285affbd37e1725043b Mon Sep 17 00:00:00 2001 From: Aaron Spring Date: Wed, 10 Mar 2021 22:46:00 +0100 Subject: [PATCH 34/39] Update probabilistic.py --- xskillscore/core/probabilistic.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index 9ee207f9..5407418c 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -532,8 +532,9 @@ def rps( of observations and forecasts data. Effectively, negative infinity is appended to the left side of category_edges, and positive infinity is appended to the right side. Thus, N category edges produces N+1 bins. For example, specifying - category_edges = [0,1] will compute the cdfs for bins [-inf, 0), [-inf, 1) and - [-inf, inf). Note that the edges are right-edge exclusive. + category_edges = [0,1] will compute the RPS for bins [-inf, 0), [0, 1) and + [1, inf), which results in CDF bins [-inf, 0), [-inf, 1) and [-inf, inf). + Note that the edges are right-edge exclusive. Forecasts, observations and category_edge are expected in absolute units or probabilities consistently. From 15923cc6a57b9d1311197754da24ee78288f2aaa Mon Sep 17 00:00:00 2001 From: Aaron Spring Date: Wed, 10 Mar 2021 22:48:25 +0100 Subject: [PATCH 35/39] Update contingency.py --- xskillscore/core/contingency.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/xskillscore/core/contingency.py b/xskillscore/core/contingency.py index 60750dd3..ae00f09c 100644 --- a/xskillscore/core/contingency.py +++ b/xskillscore/core/contingency.py @@ -13,8 +13,6 @@ def _get_category_bounds(category_edges): """Return formatted string of category bounds given list of category edges""" - if isinstance(category_edges, (xr.DataArray, xr.Dataset)): - category_edges = category_edges.category_edge.values bounds = [ f"[{str(category_edges[i])}, {str(category_edges[i + 1])})" for i in range(len(category_edges) - 2) From 6991802ec24c53ae3ee880f00871c91f39e426ff Mon Sep 17 00:00:00 2001 From: Aaron Spring Date: Wed, 10 Mar 2021 22:49:19 +0100 Subject: [PATCH 36/39] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 43df7bcb..b37423d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ bottleneck cftime -dask==2021.02.0 +dask numba>=0.52 numpy properscoring From 904a0f3f9bca1315734e6652c3e7538f9b741c40 Mon Sep 17 00:00:00 2001 From: Aaron Spring Date: Wed, 10 Mar 2021 23:00:17 +0100 Subject: [PATCH 37/39] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b37423d8..8751716f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ bottleneck cftime -dask +dask<=2021.02.0 numba>=0.52 numpy properscoring From 0267df10771958767419ad7f95f402d8e7392b1e Mon Sep 17 00:00:00 2001 From: Aaron Spring Date: Wed, 10 Mar 2021 23:02:35 +0100 Subject: [PATCH 38/39] Update probabilistic.py --- xskillscore/core/probabilistic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xskillscore/core/probabilistic.py b/xskillscore/core/probabilistic.py index 5407418c..e83ef5ad 100644 --- a/xskillscore/core/probabilistic.py +++ b/xskillscore/core/probabilistic.py @@ -533,7 +533,7 @@ def rps( to the left side of category_edges, and positive infinity is appended to the right side. Thus, N category edges produces N+1 bins. For example, specifying category_edges = [0,1] will compute the RPS for bins [-inf, 0), [0, 1) and - [1, inf), which results in CDF bins [-inf, 0), [-inf, 1) and [-inf, inf). + [1, inf), which results in CDF bins [-inf, 0), [-inf, 1) and [-inf, inf). Note that the edges are right-edge exclusive. Forecasts, observations and category_edge are expected in absolute units or probabilities consistently. From efffb791bee2a05a386cec97ed16628a5811b48a Mon Sep 17 00:00:00 2001 From: Aaron Spring Date: Wed, 10 Mar 2021 23:07:35 +0100 Subject: [PATCH 39/39] Update CHANGELOG.rst --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1facfd6a..d01f38d6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -23,7 +23,7 @@ Features - Added receiver operating characteristic (ROC) :py:func:`~xskillscore.roc`. (:issue:`114`, :issue:`256`, :pr:`236`, :pr:`259`) `Aaron Spring`_ - Added many options for ``category_edges`` in :py:func:`~xskillscore.rps`, which - allows multi-dimensional edges. :py:func:`~xskillscore.rps` now + allows multi-dimensional edges. :py:func:`~xskillscore.rps` now requires dimension ``member_dim`` in forecasts. (:issue:`275`, :pr:`277`) `Aaron Spring`_