Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix da.broadcast_to call when the fx cube has different shape than target data cube #1350

Merged
merged 3 commits into from
Oct 8, 2021
Merged

Fix da.broadcast_to call when the fx cube has different shape than target data cube #1350

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b55891d
fix the broadcasting issue in mask
valeriupredoi Oct 8, 2021
6e69ee8
make 4d data get fx masked in tests
valeriupredoi Oct 8, 2021
40b3bf3
rename fx coordinate specifications
valeriupredoi Oct 8, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions esmvalcore/preprocessor/_mask.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,8 @@ def mask_landsea(cube, mask_out, always_use_ne_mask=False):
'not found in cube. Check fx_file availability.')

if fx_cube:
fx_cube.data = da.broadcast_to(fx_cube.core_data(), cube.shape)
landsea_mask = _get_fx_mask(fx_cube.data, mask_out,
fx_cube_data = da.broadcast_to(fx_cube.core_data(), cube.shape)
landsea_mask = _get_fx_mask(fx_cube_data, mask_out,
fx_cube.var_name)
cube.data = _apply_fx_mask(landsea_mask, cube.data)
logger.debug("Applying land-sea mask: %s", fx_cube.var_name)
Expand Down Expand Up @@ -181,8 +181,8 @@ def mask_landseaice(cube, mask_out):
logger.debug('Ancillary variable land ice area fraction '
'not found in cube. Check fx_file availability.')
if fx_cube:
fx_cube.data = da.broadcast_to(fx_cube.core_data(), cube.shape)
landice_mask = _get_fx_mask(fx_cube.data, mask_out, fx_cube.var_name)
fx_cube_data = da.broadcast_to(fx_cube.core_data(), cube.shape)
landice_mask = _get_fx_mask(fx_cube_data, mask_out, fx_cube.var_name)
cube.data = _apply_fx_mask(landice_mask, cube.data)
logger.debug("Applying landsea-ice mask: sftgif")
else:
Expand Down
83 changes: 54 additions & 29 deletions tests/integration/preprocessor/_mask/test_mask.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ def setUp(self):
fx_data = np.empty((3, 3))
fx_data[:] = 60.
fx_data[1, 2] = 30.
self.new_cube_data = np.empty((3, 3))
self.new_cube_data = np.empty((2, 3, 3))
self.new_cube_data[:] = 200.
crd_sys = iris.coord_systems.GeogCS(iris.fileformats.pp.EARTH_RADIUS)
self.lons = iris.coords.DimCoord([0, 1.5, 3],
Expand All @@ -38,6 +38,11 @@ def setUp(self):
bounds=[[0, 1], [1, 2], [2, 3]],
units='degrees_north',
coord_system=crd_sys)
self.zcoord = iris.coords.DimCoord([0.5, 5.],
long_name='zcoord',
bounds=[[0., 2.5], [2.5, 25.]],
units='m',
attributes={'positive': 'down'})
self.times = iris.coords.DimCoord([0, 1.5, 2.5, 3.5],
standard_name='time',
bounds=[[0, 1], [1, 2], [2, 3],
Expand All @@ -47,9 +52,11 @@ def setUp(self):
standard_name='time',
bounds=[[0, 1], [1, 2], [2, 3]],
units='hours')
self.coords_spec = [(self.lats, 0), (self.lons, 1)]
self.fx_coords_spec = [(self.lats, 0), (self.lons, 1)]
self.cube_coords_spec = [(self.zcoord, 0),
(self.lats, 1), (self.lons, 2)]
self.fx_mask = iris.cube.Cube(fx_data,
dim_coords_and_dims=self.coords_spec,
dim_coords_and_dims=self.fx_coords_spec,
units='%')
self.mock_data = np.ma.empty((4, 3, 3))
self.mock_data[:] = 10.
Expand All @@ -69,8 +76,10 @@ def test_components_fx_var(self, tmp_path):
'frequency': 'fx',
'filename': sftlf_file}
}
new_cube_land = iris.cube.Cube(self.new_cube_data,
dim_coords_and_dims=self.coords_spec)
new_cube_land = iris.cube.Cube(
self.new_cube_data,
dim_coords_and_dims=self.cube_coords_spec
)
new_cube_land = add_fx_variables(new_cube_land, fx_vars,
CheckLevels.IGNORE)
result_land = mask_landsea(
Expand All @@ -92,8 +101,10 @@ def test_components_fx_var(self, tmp_path):
'frequency': 'fx',
'filename': sftlf_file}
}
new_cube_ice = iris.cube.Cube(self.new_cube_data,
dim_coords_and_dims=self.coords_spec)
new_cube_ice = iris.cube.Cube(
self.new_cube_data,
dim_coords_and_dims=self.cube_coords_spec
)
new_cube_ice = add_fx_variables(new_cube_ice, fx_vars,
CheckLevels.IGNORE)
result_ice = mask_landseaice(
Expand All @@ -117,12 +128,16 @@ def test_mask_landsea(self, tmp_path):
'frequency': 'fx',
'filename': sftlf_file}
}
new_cube_land = iris.cube.Cube(self.new_cube_data,
dim_coords_and_dims=self.coords_spec)
new_cube_land = iris.cube.Cube(
self.new_cube_data,
dim_coords_and_dims=self.cube_coords_spec
)
new_cube_land = add_fx_variables(new_cube_land, fx_vars,
CheckLevels.IGNORE)
new_cube_sea = iris.cube.Cube(self.new_cube_data,
dim_coords_and_dims=self.coords_spec)
new_cube_sea = iris.cube.Cube(
self.new_cube_data,
dim_coords_and_dims=self.cube_coords_spec
)
new_cube_sea = add_fx_variables(new_cube_sea, fx_vars,
CheckLevels.IGNORE)

Expand All @@ -135,26 +150,30 @@ def test_mask_landsea(self, tmp_path):
new_cube_sea,
'sea',
)
expected = np.ma.empty((3, 3))
expected = np.ma.empty((2, 3, 3))
expected.data[:] = 200.
expected.mask = np.ones((3, 3), bool)
expected.mask[1, 2] = False
expected.mask = np.ones((2, 3, 3), bool)
expected.mask[:, 1, 2] = False
# set fillvalues so we are sure they are equal
np.ma.set_fill_value(result_land.data, 1e+20)
np.ma.set_fill_value(result_sea.data, 1e+20)
np.ma.set_fill_value(expected, 1e+20)
assert_array_equal(result_land.data, expected)
expected.mask = np.zeros((3, 3), bool)
expected.mask[1, 2] = True
expected.mask = np.zeros((2, 3, 3), bool)
expected.mask[:, 1, 2] = True
assert_array_equal(result_sea.data, expected)

# Mask with shp files although sftlf is available
new_cube_land = iris.cube.Cube(self.new_cube_data,
dim_coords_and_dims=self.coords_spec)
new_cube_land = iris.cube.Cube(
self.new_cube_data,
dim_coords_and_dims=self.cube_coords_spec
)
new_cube_land = add_fx_variables(new_cube_land, fx_vars,
CheckLevels.IGNORE)
new_cube_sea = iris.cube.Cube(self.new_cube_data,
dim_coords_and_dims=self.coords_spec)
new_cube_sea = iris.cube.Cube(
self.new_cube_data,
dim_coords_and_dims=self.cube_coords_spec
)
new_cube_sea = add_fx_variables(new_cube_sea, fx_vars,
CheckLevels.IGNORE)
result_land = mask_landsea(
Expand All @@ -177,10 +196,14 @@ def test_mask_landsea(self, tmp_path):
assert_array_equal(result_sea.data, expected)

# mask with shp files
new_cube_land = iris.cube.Cube(self.new_cube_data,
dim_coords_and_dims=self.coords_spec)
new_cube_sea = iris.cube.Cube(self.new_cube_data,
dim_coords_and_dims=self.coords_spec)
new_cube_land = iris.cube.Cube(
self.new_cube_data,
dim_coords_and_dims=self.cube_coords_spec
)
new_cube_sea = iris.cube.Cube(
self.new_cube_data,
dim_coords_and_dims=self.cube_coords_spec
)
result_land = mask_landsea(new_cube_land, 'land')
result_sea = mask_landsea(new_cube_sea, 'sea')

Expand All @@ -207,15 +230,17 @@ def test_mask_landseaice(self, tmp_path):
'frequency': 'fx',
'filename': sftgif_file}
}
new_cube_ice = iris.cube.Cube(self.new_cube_data,
dim_coords_and_dims=self.coords_spec)
new_cube_ice = iris.cube.Cube(
self.new_cube_data,
dim_coords_and_dims=self.cube_coords_spec
)
new_cube_ice = add_fx_variables(new_cube_ice, fx_vars,
CheckLevels.IGNORE)
result_ice = mask_landseaice(new_cube_ice, 'ice')
expected = np.ma.empty((3, 3))
expected = np.ma.empty((2, 3, 3))
expected.data[:] = 200.
expected.mask = np.ones((3, 3), bool)
expected.mask[1, 2] = False
expected.mask = np.ones((2, 3, 3), bool)
expected.mask[:, 1, 2] = False
np.ma.set_fill_value(result_ice.data, 1e+20)
np.ma.set_fill_value(expected, 1e+20)
assert_array_equal(result_ice.data, expected)
Expand Down