From 3d0c7a291b5fdc417b534abb92b615ab5355c026 Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 28 Oct 2021 10:26:01 +0100 Subject: [PATCH 01/16] Export axes as dict of name and type for Image --- src/omero_zarr/raw_pixels.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/omero_zarr/raw_pixels.py b/src/omero_zarr/raw_pixels.py index 43de8d4..3334068 100644 --- a/src/omero_zarr/raw_pixels.py +++ b/src/omero_zarr/raw_pixels.py @@ -33,7 +33,7 @@ def image_to_zarr(image: omero.gateway.ImageWrapper, args: argparse.Namespace) - def add_image( image: omero.gateway.ImageWrapper, parent: Group, cache_dir: Optional[str] = None -) -> Tuple[int, List[str]]: +) -> Tuple[int, List[Dict[str, Any]]]: """Adds an OMERO image pixel data as array to the given parent zarr group. Optionally caches the pixel data in the given cache_dir directory. Returns the number of resolution levels generated for the image. @@ -103,7 +103,7 @@ def add_raw_image( level_count: int, cache_dir: Optional[str] = None, cache_file_name_func: Callable[[int, int, int], str] = None, -) -> Tuple[int, List[str]]: +) -> Tuple[int, List[Dict[str, Any]]]: """Adds the raw image pixel data as array to the given parent zarr group. Optionally caches the pixel data in the given cache_dir directory. Returns the number of resolution levels generated for the image. @@ -123,11 +123,11 @@ def add_raw_image( dims = [dim for dim in [size_t, size_c, size_z] if dim != 1] axes = [] if size_t > 1: - axes.append("t") + axes.append({"name": "t", "type": "time"}) if size_c > 1: - axes.append("c") + axes.append({"name": "c", "type": "channel"}) if size_z > 1: - axes.append("z") + axes.append({"name": "z", "type": "space"}) field_groups: List[Array] = [] for t in range(size_t): @@ -179,7 +179,8 @@ def add_raw_image( preserve_range=True, anti_aliasing=False, ).astype(plane.dtype) - return (level_count, axes + ["y", "x"]) + axes = axes + [{"name": "y", "type": "space"}, {"name": "x", "type": "space"}] + return (level_count, axes) def marshal_acquisition(acquisition: omero.gateway._PlateAcquisitionWrapper) -> Dict: @@ -275,7 +276,7 @@ def plate_to_zarr(plate: omero.gateway._PlateWrapper, args: argparse.Namespace) def add_multiscales_metadata( zarr_root: Group, - axes: List[str], + axes: List[Dict[str, str]], resolutions: int = 1, ) -> None: From 288a1d630d9879f59b20e1e7b4e33a6f6ad1f61d Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 28 Oct 2021 10:27:13 +0100 Subject: [PATCH 02/16] Update version to 0.4 --- src/omero_zarr/__init__.py | 2 +- src/omero_zarr/masks.py | 4 ++-- src/omero_zarr/raw_pixels.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/omero_zarr/__init__.py b/src/omero_zarr/__init__.py index 6dfa181..531ae6c 100644 --- a/src/omero_zarr/__init__.py +++ b/src/omero_zarr/__init__.py @@ -1,6 +1,6 @@ from ._version import version as __version__ -ngff_version = "0.3" +ngff_version = "0.4" __all__ = [ "__version__", diff --git a/src/omero_zarr/masks.py b/src/omero_zarr/masks.py index 921b860..6b94e36 100644 --- a/src/omero_zarr/masks.py +++ b/src/omero_zarr/masks.py @@ -303,7 +303,7 @@ def save(self, masks: List[omero.model.Shape], name: str) -> None: ignored_dimensions, check_overlaps=True, ) - # For v0.3 ngff we want to reduce the number of dimensions to + # For v0.3+ ngff we want to reduce the number of dimensions to # match the dims of the Image. dims_to_squeeze = [] axes = [] @@ -326,7 +326,7 @@ def save(self, masks: List[omero.model.Shape], name: str) -> None: image_label_colors: List[JSONDict] = [] label_properties: List[JSONDict] = [] image_label = { - "version": "0.3", + "version": "0.4", "colors": image_label_colors, "properties": label_properties, "source": {"image": source_image_link}, diff --git a/src/omero_zarr/raw_pixels.py b/src/omero_zarr/raw_pixels.py index 3334068..2283f36 100644 --- a/src/omero_zarr/raw_pixels.py +++ b/src/omero_zarr/raw_pixels.py @@ -282,7 +282,7 @@ def add_multiscales_metadata( multiscales = [ { - "version": "0.3", + "version": "0.4", "datasets": [{"path": str(r)} for r in range(resolutions)], "axes": axes, } From 9f2297e0693ad3799ddf46efb2c37520b267d60e Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 20 Dec 2021 12:36:49 +0000 Subject: [PATCH 03/16] Add units info to axes --- src/omero_zarr/raw_pixels.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/omero_zarr/raw_pixels.py b/src/omero_zarr/raw_pixels.py index 2283f36..3b98bbb 100644 --- a/src/omero_zarr/raw_pixels.py +++ b/src/omero_zarr/raw_pixels.py @@ -49,6 +49,26 @@ def get_cache_filename(z: int, c: int, t: int) -> str: size_y = image.getSizeY() size_t = image.getSizeT() d_type = image.getPixelsType() + pixel_sizes = {} + pix_size_x = image.getPixelSizeX(units=True) + pix_size_y = image.getPixelSizeY(units=True) + pix_size_z = image.getPixelSizeZ(units=True) + # All OMERO units.toLowerCase() are valid UDUNITS-2 and therefore NGFF spec + if pix_size_x is not None: + pixel_sizes["x"] = { + "units": str(pix_size_x.getUnit()).lower(), + "value": pix_size_x.getValue(), + } + if pix_size_y is not None: + pixel_sizes["y"] = { + "units": str(pix_size_y.getUnit()).lower(), + "value": pix_size_y.getValue(), + } + if pix_size_z is not None: + pixel_sizes["z"] = { + "units": str(pix_size_z.getUnit()).lower(), + "value": pix_size_z.getValue(), + } zct_list = [] for t in range(size_t): @@ -89,6 +109,7 @@ def planeGen() -> np.ndarray: level_count=level_count, cache_dir=cache_dir, cache_file_name_func=get_cache_filename, + pixel_sizes=pixel_sizes, ) @@ -103,6 +124,7 @@ def add_raw_image( level_count: int, cache_dir: Optional[str] = None, cache_file_name_func: Callable[[int, int, int], str] = None, + pixel_sizes: Dict[str, Dict[str, Any]] = None, ) -> Tuple[int, List[Dict[str, Any]]]: """Adds the raw image pixel data as array to the given parent zarr group. Optionally caches the pixel data in the given cache_dir directory. @@ -128,6 +150,8 @@ def add_raw_image( axes.append({"name": "c", "type": "channel"}) if size_z > 1: axes.append({"name": "z", "type": "space"}) + if pixel_sizes and "z" in pixel_sizes: + axes[-1]["units"] = pixel_sizes["z"]["units"] field_groups: List[Array] = [] for t in range(size_t): @@ -179,7 +203,12 @@ def add_raw_image( preserve_range=True, anti_aliasing=False, ).astype(plane.dtype) - axes = axes + [{"name": "y", "type": "space"}, {"name": "x", "type": "space"}] + + # last 2 dimensions are always y and x + for dim in ("y", "x"): + axes.append({"name": dim, "type": "space"}) + if pixel_sizes and dim in pixel_sizes: + axes[-1]["units"] = pixel_sizes[dim]["units"] return (level_count, axes) From d079905e93eac27f0455d271b33f60125529119c Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 20 Dec 2021 12:46:53 +0000 Subject: [PATCH 04/16] Move axes creation to add_image() --- src/omero_zarr/raw_pixels.py | 39 ++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/omero_zarr/raw_pixels.py b/src/omero_zarr/raw_pixels.py index 3b98bbb..bde544e 100644 --- a/src/omero_zarr/raw_pixels.py +++ b/src/omero_zarr/raw_pixels.py @@ -70,6 +70,21 @@ def get_cache_filename(z: int, c: int, t: int) -> str: "value": pix_size_z.getValue(), } + axes = [] + if size_t > 1: + axes.append({"name": "t", "type": "time"}) + if size_c > 1: + axes.append({"name": "c", "type": "channel"}) + if size_z > 1: + axes.append({"name": "z", "type": "space"}) + if pixel_sizes and "z" in pixel_sizes: + axes[-1]["units"] = pixel_sizes["z"]["units"] + # last 2 dimensions are always y and x + for dim in ("y", "x"): + axes.append({"name": dim, "type": "space"}) + if pixel_sizes and dim in pixel_sizes: + axes[-1]["units"] = pixel_sizes[dim]["units"] + zct_list = [] for t in range(size_t): for c in range(size_c): @@ -99,7 +114,7 @@ def planeGen() -> np.ndarray: longest = longest // 2 level_count += 1 - return add_raw_image( + level_count = add_raw_image( planes=planes, size_z=size_z, size_c=size_c, @@ -109,9 +124,10 @@ def planeGen() -> np.ndarray: level_count=level_count, cache_dir=cache_dir, cache_file_name_func=get_cache_filename, - pixel_sizes=pixel_sizes, ) + return (level_count, axes) + def add_raw_image( *, @@ -124,8 +140,7 @@ def add_raw_image( level_count: int, cache_dir: Optional[str] = None, cache_file_name_func: Callable[[int, int, int], str] = None, - pixel_sizes: Dict[str, Dict[str, Any]] = None, -) -> Tuple[int, List[Dict[str, Any]]]: +) -> int: """Adds the raw image pixel data as array to the given parent zarr group. Optionally caches the pixel data in the given cache_dir directory. Returns the number of resolution levels generated for the image. @@ -143,15 +158,6 @@ def add_raw_image( cache_dir = "" dims = [dim for dim in [size_t, size_c, size_z] if dim != 1] - axes = [] - if size_t > 1: - axes.append({"name": "t", "type": "time"}) - if size_c > 1: - axes.append({"name": "c", "type": "channel"}) - if size_z > 1: - axes.append({"name": "z", "type": "space"}) - if pixel_sizes and "z" in pixel_sizes: - axes[-1]["units"] = pixel_sizes["z"]["units"] field_groups: List[Array] = [] for t in range(size_t): @@ -204,12 +210,7 @@ def add_raw_image( anti_aliasing=False, ).astype(plane.dtype) - # last 2 dimensions are always y and x - for dim in ("y", "x"): - axes.append({"name": dim, "type": "space"}) - if pixel_sizes and dim in pixel_sizes: - axes[-1]["units"] = pixel_sizes[dim]["units"] - return (level_count, axes) + return level_count def marshal_acquisition(acquisition: omero.gateway._PlateAcquisitionWrapper) -> Dict: From b3db05df7dfcb326dc406c0176f453c870333db6 Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 20 Dec 2021 14:40:23 +0000 Subject: [PATCH 05/16] add_image() also does add_multiscales_metadata() --- src/omero_zarr/raw_pixels.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/omero_zarr/raw_pixels.py b/src/omero_zarr/raw_pixels.py index bde544e..993eb28 100644 --- a/src/omero_zarr/raw_pixels.py +++ b/src/omero_zarr/raw_pixels.py @@ -24,8 +24,7 @@ def image_to_zarr(image: omero.gateway.ImageWrapper, args: argparse.Namespace) - print(f"Exporting to {name} ({VERSION})") store = open_store(name) root = open_group(store) - n_levels, axes = add_image(image, root, cache_dir=cache_dir) - add_multiscales_metadata(root, axes, n_levels) + add_image(image, root, cache_dir=cache_dir) add_omero_metadata(root, image) add_toplevel_metadata(root) print("Finished.") @@ -53,7 +52,7 @@ def get_cache_filename(z: int, c: int, t: int) -> str: pix_size_x = image.getPixelSizeX(units=True) pix_size_y = image.getPixelSizeY(units=True) pix_size_z = image.getPixelSizeZ(units=True) - # All OMERO units.toLowerCase() are valid UDUNITS-2 and therefore NGFF spec + # All OMERO units.lower() are valid UDUNITS-2 and therefore NGFF spec if pix_size_x is not None: pixel_sizes["x"] = { "units": str(pix_size_x.getUnit()).lower(), @@ -126,6 +125,8 @@ def planeGen() -> np.ndarray: cache_file_name_func=get_cache_filename, ) + add_multiscales_metadata(parent, axes, level_count) + return (level_count, axes) @@ -287,8 +288,7 @@ def plate_to_zarr(plate: omero.gateway._PlateWrapper, args: argparse.Namespace) row_group = root.require_group(row) col_group = row_group.require_group(col) field_group = col_group.require_group(field_name) - n_levels, axes = add_image(img, field_group, cache_dir=cache_dir) - add_multiscales_metadata(field_group, axes, n_levels) + add_image(img, field_group, cache_dir=cache_dir) add_omero_metadata(field_group, img) # Update Well metadata after each image col_group.attrs["well"] = {"images": fields, "version": VERSION} From 9568de985f01683fc255239135fec0faf4b07255 Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 20 Dec 2021 15:25:01 +0000 Subject: [PATCH 06/16] Define path = str(level) in 1 place only --- src/omero_zarr/raw_pixels.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/omero_zarr/raw_pixels.py b/src/omero_zarr/raw_pixels.py index 993eb28..93d9418 100644 --- a/src/omero_zarr/raw_pixels.py +++ b/src/omero_zarr/raw_pixels.py @@ -113,7 +113,7 @@ def planeGen() -> np.ndarray: longest = longest // 2 level_count += 1 - level_count = add_raw_image( + paths = add_raw_image( planes=planes, size_z=size_z, size_c=size_c, @@ -125,7 +125,7 @@ def planeGen() -> np.ndarray: cache_file_name_func=get_cache_filename, ) - add_multiscales_metadata(parent, axes, level_count) + add_multiscales_metadata(parent, axes, paths) return (level_count, axes) @@ -141,7 +141,7 @@ def add_raw_image( level_count: int, cache_dir: Optional[str] = None, cache_file_name_func: Callable[[int, int, int], str] = None, -) -> int: +) -> List[str]: """Adds the raw image pixel data as array to the given parent zarr group. Optionally caches the pixel data in the given cache_dir directory. Returns the number of resolution levels generated for the image. @@ -160,6 +160,7 @@ def add_raw_image( dims = [dim for dim in [size_t, size_c, size_z] if dim != 1] + paths: List[str] = [] field_groups: List[Array] = [] for t in range(size_t): for c in range(size_c): @@ -182,9 +183,11 @@ def add_raw_image( size_x = plane.shape[1] # If on first plane, create a new group for this resolution level if len(field_groups) <= level: + path = str(level) + paths.append(path) field_groups.append( parent.create( - str(level), + path, shape=tuple(dims + [size_y, size_x]), chunks=tuple([1] * len(dims) + [size_y, size_x]), dtype=d_type, @@ -211,7 +214,7 @@ def add_raw_image( anti_aliasing=False, ).astype(plane.dtype) - return level_count + return paths def marshal_acquisition(acquisition: omero.gateway._PlateAcquisitionWrapper) -> Dict: @@ -307,13 +310,13 @@ def plate_to_zarr(plate: omero.gateway._PlateWrapper, args: argparse.Namespace) def add_multiscales_metadata( zarr_root: Group, axes: List[Dict[str, str]], - resolutions: int = 1, + paths: List[str], ) -> None: multiscales = [ { "version": "0.4", - "datasets": [{"path": str(r)} for r in range(resolutions)], + "datasets": [{"path": path} for path in paths], "axes": axes, } ] From 3ec5f569c31a160ba56e1aa14a92c638f478a38a Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 20 Dec 2021 16:08:33 +0000 Subject: [PATCH 07/16] Add 'transformations' support for x, y, z --- src/omero_zarr/raw_pixels.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/omero_zarr/raw_pixels.py b/src/omero_zarr/raw_pixels.py index 93d9418..a66746d 100644 --- a/src/omero_zarr/raw_pixels.py +++ b/src/omero_zarr/raw_pixels.py @@ -125,7 +125,26 @@ def planeGen() -> np.ndarray: cache_file_name_func=get_cache_filename, ) - add_multiscales_metadata(parent, axes, paths) + # Each path needs a transformations list... + transformations = [] + zooms = {"x": 1, "y": 1, "z": 1} + for path in paths: + # {"type": "scale", "scale": [2.0, 2.0, 2.0], "axisIndices": [2, 3, 4]} + scales = [] + axisIndices = [] + for index, axis in enumerate(axes): + if axis["name"] in pixel_sizes: + scales.append(zooms[axis["name"]] * pixel_sizes[axis["name"]]["value"]) + axisIndices.append(index) + # ...with a single 'scale' transformation each + transformations.append( + [{"type": "scale", "scale": scales, "axisIndices": axisIndices}] + ) + # NB we rescale X and Y for each level, but not Z + zooms["x"] = zooms["x"] * 2 + zooms["y"] = zooms["y"] * 2 + + add_multiscales_metadata(parent, axes, paths, transformations) return (level_count, axes) @@ -311,12 +330,18 @@ def add_multiscales_metadata( zarr_root: Group, axes: List[Dict[str, str]], paths: List[str], + transformations: List[List[Dict[str, Any]]] = None, ) -> None: + # transformations is a 2D list. For each path, we have a List of Dicts + datasets: List[Dict[str, Any]] = [{"path": path} for path in paths] + if transformations is not None: + for index, transform in enumerate(transformations): + datasets[index]["transformations"] = transform multiscales = [ { "version": "0.4", - "datasets": [{"path": path} for path in paths], + "datasets": datasets, "axes": axes, } ] From 70975a441943f463a7d15766441244b3d65d4d96 Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 18 Jan 2022 23:24:45 +0000 Subject: [PATCH 08/16] Use ome-zarr-py write_multiscales_metadata() --- src/omero_zarr/raw_pixels.py | 34 ++++++++-------------------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/src/omero_zarr/raw_pixels.py b/src/omero_zarr/raw_pixels.py index a66746d..10d3c73 100644 --- a/src/omero_zarr/raw_pixels.py +++ b/src/omero_zarr/raw_pixels.py @@ -7,6 +7,7 @@ import numpy as np import omero.clients # noqa import omero.gateway # required to allow 'from omero_zarr import raw_pixels' +from ome_zarr.writer import write_multiscales_metadata from omero.rtypes import unwrap from skimage.transform import resize from zarr.hierarchy import Array, Group, open_group @@ -137,14 +138,17 @@ def planeGen() -> np.ndarray: scales.append(zooms[axis["name"]] * pixel_sizes[axis["name"]]["value"]) axisIndices.append(index) # ...with a single 'scale' transformation each - transformations.append( - [{"type": "scale", "scale": scales, "axisIndices": axisIndices}] - ) + if len(scales) > 0: + transformations.append( + [{"type": "scale", "scale": scales, "axisIndices": axisIndices}] + ) # NB we rescale X and Y for each level, but not Z zooms["x"] = zooms["x"] * 2 zooms["y"] = zooms["y"] * 2 - add_multiscales_metadata(parent, axes, paths, transformations) + write_multiscales_metadata( + parent, paths, axes=axes, transformations=transformations + ) return (level_count, axes) @@ -326,28 +330,6 @@ def plate_to_zarr(plate: omero.gateway._PlateWrapper, args: argparse.Namespace) print("Finished.") -def add_multiscales_metadata( - zarr_root: Group, - axes: List[Dict[str, str]], - paths: List[str], - transformations: List[List[Dict[str, Any]]] = None, -) -> None: - # transformations is a 2D list. For each path, we have a List of Dicts - datasets: List[Dict[str, Any]] = [{"path": path} for path in paths] - if transformations is not None: - for index, transform in enumerate(transformations): - datasets[index]["transformations"] = transform - - multiscales = [ - { - "version": "0.4", - "datasets": datasets, - "axes": axes, - } - ] - zarr_root.attrs["multiscales"] = multiscales - - def add_omero_metadata(zarr_root: Group, image: omero.gateway.ImageWrapper) -> None: image_data = { From cc44688116add04e1cebe3cbb4cdbf374a4c629e Mon Sep 17 00:00:00 2001 From: William Moore Date: Wed, 19 Jan 2022 12:40:53 +0000 Subject: [PATCH 09/16] Use ome-zarr .write_plate_metadata() --- src/omero_zarr/raw_pixels.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/omero_zarr/raw_pixels.py b/src/omero_zarr/raw_pixels.py index 10d3c73..c77f8b5 100644 --- a/src/omero_zarr/raw_pixels.py +++ b/src/omero_zarr/raw_pixels.py @@ -7,7 +7,7 @@ import numpy as np import omero.clients # noqa import omero.gateway # required to allow 'from omero_zarr import raw_pixels' -from ome_zarr.writer import write_multiscales_metadata +from ome_zarr.writer import write_multiscales_metadata, write_plate_metadata from omero.rtypes import unwrap from skimage.transform import resize from zarr.hierarchy import Array, Group, open_group @@ -280,20 +280,14 @@ def plate_to_zarr(plate: omero.gateway._PlateWrapper, args: argparse.Namespace) well_paths = set() - col_names = plate.getColumnLabels() - row_names = plate.getRowLabels() + col_names = [str(name) for name in plate.getColumnLabels()] + row_names = [str(name) for name in plate.getRowLabels()] - plate_metadata = { - "name": plate.name, - "rows": [{"name": str(name)} for name in row_names], - "columns": [{"name": str(name)} for name in col_names], - "version": VERSION, - } # Add acquisitions key if at least one plate acquisition exists acquisitions = list(plate.listPlateAcquisitions()) + plate_acq = None if acquisitions: - plate_metadata["acquisitions"] = [marshal_acquisition(x) for x in acquisitions] - root.attrs["plate"] = plate_metadata + plate_acq = [marshal_acquisition(x) for x in acquisitions] for well in plate.listChildren(): row = plate.getRowLabels()[well.row] @@ -322,9 +316,15 @@ def plate_to_zarr(plate: omero.gateway._PlateWrapper, args: argparse.Namespace) print_status(int(t0), int(time.time()), count, total) # Update plate_metadata after each Well - plate_metadata["wells"] = [{"path": x} for x in well_paths] - plate_metadata["field_count"] = max_fields - root.attrs["plate"] = plate_metadata + write_plate_metadata( + root, + row_names, + col_names, + wells=list(well_paths), + field_count=max_fields, + acquisitions=plate_acq, + name=plate.name, + ) add_toplevel_metadata(root) print("Finished.") From b4ecba4ca6707da698122bd5e4c589b3e05a67aa Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 20 Jan 2022 18:38:14 +0000 Subject: [PATCH 10/16] use same axes and transf creation for image and labels --- src/omero_zarr/masks.py | 11 +++--- src/omero_zarr/raw_pixels.py | 65 +++------------------------------- src/omero_zarr/util.py | 68 ++++++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 65 deletions(-) diff --git a/src/omero_zarr/masks.py b/src/omero_zarr/masks.py index 6b94e36..a877286 100644 --- a/src/omero_zarr/masks.py +++ b/src/omero_zarr/masks.py @@ -18,7 +18,7 @@ from skimage.draw import polygon as sk_polygon from zarr.hierarchy import open_group -from .util import open_store, print_status +from .util import marshal_axes, open_store, print_status # Mapping of dimension names to axes in the Zarr DIMENSION_ORDER: Dict[str, int] = { @@ -215,6 +215,7 @@ def set_image( :param plate_path: The zarr path to the image :return: None """ + self.image = image self.size_t = image.getSizeT() self.size_c = image.getSizeC() self.size_z = image.getSizeZ() @@ -303,15 +304,15 @@ def save(self, masks: List[omero.model.Shape], name: str) -> None: ignored_dimensions, check_overlaps=True, ) + + metadata = marshal_axes(self.image, levels=1) + # For v0.3+ ngff we want to reduce the number of dimensions to # match the dims of the Image. dims_to_squeeze = [] - axes = [] for dim, size in enumerate(self.image_shape): if size == 1: dims_to_squeeze.append(dim) - else: - axes.append("tczyx"[dim]) labels = np.squeeze(labels, axis=tuple(dims_to_squeeze)) scaler = Scaler(max_layer=input_pyramid_levels) @@ -319,7 +320,7 @@ def save(self, masks: List[omero.model.Shape], name: str) -> None: pyramid_grp = out_labels.require_group(name) write_multiscale( - label_pyramid, pyramid_grp, axes=axes + label_pyramid, pyramid_grp, **metadata ) # TODO: dtype, chunks, overwite # Specify and store metadata diff --git a/src/omero_zarr/raw_pixels.py b/src/omero_zarr/raw_pixels.py index c77f8b5..a646fc5 100644 --- a/src/omero_zarr/raw_pixels.py +++ b/src/omero_zarr/raw_pixels.py @@ -14,7 +14,7 @@ from . import __version__ from . import ngff_version as VERSION -from .util import open_store, print_status +from .util import marshal_axes, open_store, print_status def image_to_zarr(image: omero.gateway.ImageWrapper, args: argparse.Namespace) -> None: @@ -49,41 +49,6 @@ def get_cache_filename(z: int, c: int, t: int) -> str: size_y = image.getSizeY() size_t = image.getSizeT() d_type = image.getPixelsType() - pixel_sizes = {} - pix_size_x = image.getPixelSizeX(units=True) - pix_size_y = image.getPixelSizeY(units=True) - pix_size_z = image.getPixelSizeZ(units=True) - # All OMERO units.lower() are valid UDUNITS-2 and therefore NGFF spec - if pix_size_x is not None: - pixel_sizes["x"] = { - "units": str(pix_size_x.getUnit()).lower(), - "value": pix_size_x.getValue(), - } - if pix_size_y is not None: - pixel_sizes["y"] = { - "units": str(pix_size_y.getUnit()).lower(), - "value": pix_size_y.getValue(), - } - if pix_size_z is not None: - pixel_sizes["z"] = { - "units": str(pix_size_z.getUnit()).lower(), - "value": pix_size_z.getValue(), - } - - axes = [] - if size_t > 1: - axes.append({"name": "t", "type": "time"}) - if size_c > 1: - axes.append({"name": "c", "type": "channel"}) - if size_z > 1: - axes.append({"name": "z", "type": "space"}) - if pixel_sizes and "z" in pixel_sizes: - axes[-1]["units"] = pixel_sizes["z"]["units"] - # last 2 dimensions are always y and x - for dim in ("y", "x"): - axes.append({"name": dim, "type": "space"}) - if pixel_sizes and dim in pixel_sizes: - axes[-1]["units"] = pixel_sizes[dim]["units"] zct_list = [] for t in range(size_t): @@ -126,31 +91,11 @@ def planeGen() -> np.ndarray: cache_file_name_func=get_cache_filename, ) - # Each path needs a transformations list... - transformations = [] - zooms = {"x": 1, "y": 1, "z": 1} - for path in paths: - # {"type": "scale", "scale": [2.0, 2.0, 2.0], "axisIndices": [2, 3, 4]} - scales = [] - axisIndices = [] - for index, axis in enumerate(axes): - if axis["name"] in pixel_sizes: - scales.append(zooms[axis["name"]] * pixel_sizes[axis["name"]]["value"]) - axisIndices.append(index) - # ...with a single 'scale' transformation each - if len(scales) > 0: - transformations.append( - [{"type": "scale", "scale": scales, "axisIndices": axisIndices}] - ) - # NB we rescale X and Y for each level, but not Z - zooms["x"] = zooms["x"] * 2 - zooms["y"] = zooms["y"] * 2 - - write_multiscales_metadata( - parent, paths, axes=axes, transformations=transformations - ) + metadata = marshal_axes(image, len(paths)) + + write_multiscales_metadata(parent, paths, **metadata) - return (level_count, axes) + return (level_count, metadata["axes"]) def add_raw_image( diff --git a/src/omero_zarr/util.py b/src/omero_zarr/util.py index 7148ead..3d9bb2b 100644 --- a/src/omero_zarr/util.py +++ b/src/omero_zarr/util.py @@ -1,5 +1,7 @@ import time +from typing import Dict, List +from omero.gateway import ImageWrapper from zarr.storage import FSStore @@ -33,3 +35,69 @@ def open_store(name: str) -> FSStore: normalize_keys=False, mode="w", ) + + +def marshal_axes( + image: ImageWrapper, levels: int = 1, multiscales_zoom: float = 2.0 +) -> Dict[str, List]: + # Prepare axes and transformations info... + size_c = image.getSizeC() + size_z = image.getSizeZ() + size_t = image.getSizeT() + pixel_sizes = {} + pix_size_x = image.getPixelSizeX(units=True) + pix_size_y = image.getPixelSizeY(units=True) + pix_size_z = image.getPixelSizeZ(units=True) + # All OMERO units.lower() are valid UDUNITS-2 and therefore NGFF spec + if pix_size_x is not None: + pixel_sizes["x"] = { + "units": str(pix_size_x.getUnit()).lower(), + "value": pix_size_x.getValue(), + } + if pix_size_y is not None: + pixel_sizes["y"] = { + "units": str(pix_size_y.getUnit()).lower(), + "value": pix_size_y.getValue(), + } + if pix_size_z is not None: + pixel_sizes["z"] = { + "units": str(pix_size_z.getUnit()).lower(), + "value": pix_size_z.getValue(), + } + + axes = [] + if size_t > 1: + axes.append({"name": "t", "type": "time"}) + if size_c > 1: + axes.append({"name": "c", "type": "channel"}) + if size_z > 1: + axes.append({"name": "z", "type": "space"}) + if pixel_sizes and "z" in pixel_sizes: + axes[-1]["units"] = pixel_sizes["z"]["units"] + # last 2 dimensions are always y and x + for dim in ("y", "x"): + axes.append({"name": dim, "type": "space"}) + if pixel_sizes and dim in pixel_sizes: + axes[-1]["units"] = pixel_sizes[dim]["units"] + + # Each path needs a transformations list... + transformations = [] + zooms = {"x": 1.0, "y": 1.0, "z": 1.0} + for level in range(levels): + # {"type": "scale", "scale": [2.0, 2.0, 2.0], "axisIndices": [2, 3, 4]} + scales = [] + axisIndices = [] + for index, axis in enumerate(axes): + if axis["name"] in pixel_sizes: + scales.append(zooms[axis["name"]] * pixel_sizes[axis["name"]]["value"]) + axisIndices.append(index) + # ...with a single 'scale' transformation each + if len(scales) > 0: + transformations.append( + [{"type": "scale", "scale": scales, "axisIndices": axisIndices}] + ) + # NB we rescale X and Y for each level, but not Z + zooms["x"] = zooms["x"] * multiscales_zoom + zooms["y"] = zooms["y"] * multiscales_zoom + + return {"axes": axes, "transformations": transformations} From 9194fdd62ab7017baa3491911f81c1f3877c2d19 Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 25 Jan 2022 13:31:53 +0000 Subject: [PATCH 11/16] ngff_version = CurrentFormat().version --- src/omero_zarr/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/omero_zarr/__init__.py b/src/omero_zarr/__init__.py index 531ae6c..1b7ddc0 100644 --- a/src/omero_zarr/__init__.py +++ b/src/omero_zarr/__init__.py @@ -1,6 +1,8 @@ +from ome_zarr.format import CurrentFormat + from ._version import version as __version__ -ngff_version = "0.4" +ngff_version = CurrentFormat().version __all__ = [ "__version__", From 27df32b8e54a4ca225760965fb77f024e01c8cd4 Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 25 Jan 2022 13:32:46 +0000 Subject: [PATCH 12/16] Use ome_zarr.writer.write_well_metadata() --- src/omero_zarr/raw_pixels.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/omero_zarr/raw_pixels.py b/src/omero_zarr/raw_pixels.py index a646fc5..358f026 100644 --- a/src/omero_zarr/raw_pixels.py +++ b/src/omero_zarr/raw_pixels.py @@ -7,7 +7,11 @@ import numpy as np import omero.clients # noqa import omero.gateway # required to allow 'from omero_zarr import raw_pixels' -from ome_zarr.writer import write_multiscales_metadata, write_plate_metadata +from ome_zarr.writer import ( + write_multiscales_metadata, + write_plate_metadata, + write_well_metadata, +) from omero.rtypes import unwrap from skimage.transform import resize from zarr.hierarchy import Array, Group, open_group @@ -256,7 +260,7 @@ def plate_to_zarr(plate: omero.gateway._PlateWrapper, args: argparse.Namespace) add_image(img, field_group, cache_dir=cache_dir) add_omero_metadata(field_group, img) # Update Well metadata after each image - col_group.attrs["well"] = {"images": fields, "version": VERSION} + write_well_metadata(col_group, fields) max_fields = max(max_fields, field + 1) print_status(int(t0), int(time.time()), count, total) From c9447beee39c093d82554d6d908ab257a49c72df Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 25 Jan 2022 13:37:21 +0000 Subject: [PATCH 13/16] Remove hard-coded version in masks.py --- src/omero_zarr/masks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/omero_zarr/masks.py b/src/omero_zarr/masks.py index a877286..5132ae4 100644 --- a/src/omero_zarr/masks.py +++ b/src/omero_zarr/masks.py @@ -18,6 +18,7 @@ from skimage.draw import polygon as sk_polygon from zarr.hierarchy import open_group +from . import ngff_version as VERSION from .util import marshal_axes, open_store, print_status # Mapping of dimension names to axes in the Zarr @@ -327,7 +328,7 @@ def save(self, masks: List[omero.model.Shape], name: str) -> None: image_label_colors: List[JSONDict] = [] label_properties: List[JSONDict] = [] image_label = { - "version": "0.4", + "version": VERSION, "colors": image_label_colors, "properties": label_properties, "source": {"image": source_image_link}, From 0346685ac9b82dcd1333a794cd929de834c45c79 Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 25 Jan 2022 14:59:48 +0000 Subject: [PATCH 14/16] Split marshal_transformations() into separate function --- src/omero_zarr/raw_pixels.py | 11 +++++++---- src/omero_zarr/util.py | 32 +++++++++++++++++++++++--------- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/src/omero_zarr/raw_pixels.py b/src/omero_zarr/raw_pixels.py index 358f026..566d633 100644 --- a/src/omero_zarr/raw_pixels.py +++ b/src/omero_zarr/raw_pixels.py @@ -18,7 +18,7 @@ from . import __version__ from . import ngff_version as VERSION -from .util import marshal_axes, open_store, print_status +from .util import marshal_axes, marshal_transformations, open_store, print_status def image_to_zarr(image: omero.gateway.ImageWrapper, args: argparse.Namespace) -> None: @@ -95,11 +95,14 @@ def planeGen() -> np.ndarray: cache_file_name_func=get_cache_filename, ) - metadata = marshal_axes(image, len(paths)) + axes = marshal_axes(image) + transformations = marshal_transformations(image, len(paths)) - write_multiscales_metadata(parent, paths, **metadata) + write_multiscales_metadata( + parent, paths, axes=axes, transformations=transformations + ) - return (level_count, metadata["axes"]) + return (level_count, axes) def add_raw_image( diff --git a/src/omero_zarr/util.py b/src/omero_zarr/util.py index 3d9bb2b..3d35382 100644 --- a/src/omero_zarr/util.py +++ b/src/omero_zarr/util.py @@ -37,14 +37,9 @@ def open_store(name: str) -> FSStore: ) -def marshal_axes( - image: ImageWrapper, levels: int = 1, multiscales_zoom: float = 2.0 -) -> Dict[str, List]: - # Prepare axes and transformations info... - size_c = image.getSizeC() - size_z = image.getSizeZ() - size_t = image.getSizeT() - pixel_sizes = {} +def marshal_pixel_sizes(image: ImageWrapper) -> Dict[str, Dict]: + + pixel_sizes: Dict[str, Dict] = {} pix_size_x = image.getPixelSizeX(units=True) pix_size_y = image.getPixelSizeY(units=True) pix_size_z = image.getPixelSizeZ(units=True) @@ -64,6 +59,15 @@ def marshal_axes( "units": str(pix_size_z.getUnit()).lower(), "value": pix_size_z.getValue(), } + return pixel_sizes + + +def marshal_axes(image: ImageWrapper) -> List[Dict]: + # Prepare axes and transformations info... + size_c = image.getSizeC() + size_z = image.getSizeZ() + size_t = image.getSizeT() + pixel_sizes = marshal_pixel_sizes(image) axes = [] if size_t > 1: @@ -80,6 +84,16 @@ def marshal_axes( if pixel_sizes and dim in pixel_sizes: axes[-1]["units"] = pixel_sizes[dim]["units"] + return axes + + +def marshal_transformations( + image: ImageWrapper, levels: int = 1, multiscales_zoom: float = 2.0 +) -> List[List[Dict]]: + + axes = marshal_axes(image) + pixel_sizes = marshal_pixel_sizes(image) + # Each path needs a transformations list... transformations = [] zooms = {"x": 1.0, "y": 1.0, "z": 1.0} @@ -100,4 +114,4 @@ def marshal_axes( zooms["x"] = zooms["x"] * multiscales_zoom zooms["y"] = zooms["y"] * multiscales_zoom - return {"axes": axes, "transformations": transformations} + return transformations From b4ab862e429f6f249da16d4d257677d2cfef776f Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 25 Jan 2022 15:55:05 +0000 Subject: [PATCH 15/16] Fix marshal_axes/transformations in MaskSaver --- src/omero_zarr/masks.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/omero_zarr/masks.py b/src/omero_zarr/masks.py index 5132ae4..83c5dcd 100644 --- a/src/omero_zarr/masks.py +++ b/src/omero_zarr/masks.py @@ -19,7 +19,7 @@ from zarr.hierarchy import open_group from . import ngff_version as VERSION -from .util import marshal_axes, open_store, print_status +from .util import marshal_axes, marshal_transformations, open_store, print_status # Mapping of dimension names to axes in the Zarr DIMENSION_ORDER: Dict[str, int] = { @@ -306,7 +306,8 @@ def save(self, masks: List[omero.model.Shape], name: str) -> None: check_overlaps=True, ) - metadata = marshal_axes(self.image, levels=1) + axes = marshal_axes(self.image) + transformations = marshal_transformations(self.image, levels=1) # For v0.3+ ngff we want to reduce the number of dimensions to # match the dims of the Image. @@ -321,7 +322,7 @@ def save(self, masks: List[omero.model.Shape], name: str) -> None: pyramid_grp = out_labels.require_group(name) write_multiscale( - label_pyramid, pyramid_grp, **metadata + label_pyramid, pyramid_grp, axes=axes, transformations=transformations ) # TODO: dtype, chunks, overwite # Specify and store metadata From adbe8f52fa4ac82f516e6bbfdfa52577aef1d340 Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 25 Jan 2022 15:57:07 +0000 Subject: [PATCH 16/16] setup.py requires ome-zarr>=0.3a1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cb783f3..421625b 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ def get_long_description() -> str: author="The Open Microscopy Team", author_email="", python_requires=">=3", - install_requires=["omero-py>=5.6.0", "ome-zarr>=0.2.0"], + install_requires=["omero-py>=5.6.0", "ome-zarr>=0.3a1"], long_description=long_description, keywords=["OMERO.CLI", "plugin"], url="https://github.com/ome/omero-cli-zarr/",