From 0a62ec2ca3730f0abcb7bc0ad6f675221a798e3d Mon Sep 17 00:00:00 2001 From: Igor Davidyuk Date: Wed, 29 May 2024 14:24:11 +0200 Subject: [PATCH 1/7] initial implementation Signed-off-by: Igor Davidyuk --- geti_sdk/data_models/annotation_scene.py | 23 ++ geti_sdk/data_models/utils.py | 7 +- geti_sdk/deployment/deployed_model.py | 118 +--------- .../results_to_prediction_converter.py | 201 +++++++++++++----- .../prediction_visualization/visualizer.py | 38 ++++ 5 files changed, 224 insertions(+), 163 deletions(-) diff --git a/geti_sdk/data_models/annotation_scene.py b/geti_sdk/data_models/annotation_scene.py index 20433a1b..df91b5f6 100644 --- a/geti_sdk/data_models/annotation_scene.py +++ b/geti_sdk/data_models/annotation_scene.py @@ -367,6 +367,29 @@ def map_labels( modified=self.modified, ) + def filter_annotations( + self, labels: Sequence[Union[Label, ScoredLabel]] + ) -> "AnnotationScene": + """ + Filter annotations in the scene to only include labels that are present in the + provided list of labels. + + :param labels: List of labels to filter the scene with + :return: AnnotationScene with filtered annotations + """ + label_names_to_keep = {label.name for label in labels} + filtered_annotations: List[Annotation] = [] + for annotation in self.annotations: + for label_name in annotation.label_names: + if label_name in label_names_to_keep: + filtered_annotations.append(annotation) + break + return AnnotationScene( + annotations=filtered_annotations, + media_identifier=self.media_identifier, + modified=self.modified, + ) + def resolve_label_names_and_colors(self, labels: List[Label]) -> None: """ Add label names and colors to all annotations, based on a list of available diff --git a/geti_sdk/data_models/utils.py b/geti_sdk/data_models/utils.py index e18b0944..8685ccdc 100644 --- a/geti_sdk/data_models/utils.py +++ b/geti_sdk/data_models/utils.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions # and limitations under the License. import logging +from collections.abc import Sequence from datetime import datetime from enum import Enum from typing import Any, Callable, Dict, List, Optional, Type, TypeVar, Union @@ -306,11 +307,11 @@ def remove_null_fields(input: Any): for key, value in list(input.items()): if isinstance(value, dict): remove_null_fields(value) - elif value is None or value == "": - input.pop(key) - elif isinstance(value, list): + elif isinstance(value, Sequence): for item in value: remove_null_fields(item) + elif value is None or value == "": + input.pop(key) elif isinstance(input, list): for item in input: remove_null_fields(item) diff --git a/geti_sdk/deployment/deployed_model.py b/geti_sdk/deployment/deployed_model.py index 344e5fbd..63fbd88b 100644 --- a/geti_sdk/deployment/deployed_model.py +++ b/geti_sdk/deployment/deployed_model.py @@ -88,6 +88,7 @@ def __attrs_post_init__(self): Initialize private attributes """ super().__attrs_post_init__() + self._domain: Optional[Domain] = None self._model_data_path: Optional[str] = None self._model_python_path: Optional[str] = None self._needs_tempdir_deletion: bool = False @@ -300,6 +301,7 @@ def load_inference_model( self._converter = ConverterFactory.create_converter( self.label_schema, configuration ) + self._domain = ConverterFactory._get_labels_domain(self.label_schema) model = model_api_Model.create_model( model=model_adapter, @@ -321,11 +323,9 @@ def load_inference_model( if enable_tiling: logging.info("Tiling is enabled for this model, initializing Tiler") - tiler_type = TILER_MAPPING.get(self._converter.domain, None) + tiler_type = TILER_MAPPING.get(self._domain, None) if tiler_type is None: - raise ValueError( - f"Tiling is not supported for domain {self._converter.domain}" - ) + raise ValueError(f"Tiling is not supported for domain {self._domain}") self._tiler = tiler_type(model=model, execution_mode="sync") self._tiling_enabled = True @@ -489,99 +489,6 @@ def _postprocess( """ return self._inference_model.postprocess(inference_results, metadata) - def _postprocess_explain_outputs( - self, - inference_results: Dict[str, np.ndarray], - metadata: Optional[Dict[str, Any]] = None, - ) -> Tuple[np.ndarray, np.ndarray]: - """ - Postprocess the model outputs to obtain saliency maps, feature vectors and - active scores. - - :param inference_results: Dictionary holding the results of inference - :param metadata: Dictionary holding metadata - :return: Tuple containing postprocessed outputs, formatted as follows: - - Numpy array containing the saliency map - - Numpy array containing the feature vector - """ - if self._saliency_location is None and self._feature_vector_location is None: - # When running postprocessing for the first time, we need to determine the - # key and location of the saliency map and feature vector. - if hasattr(self._inference_model, "postprocess_aux_outputs"): - ( - _, - saliency_map, - repr_vector, - _, - ) = self._inference_model.postprocess_aux_outputs( - inference_results, metadata - ) - self._saliency_location = "aux" - self._feature_vector_location = "aux" - else: - # Check all possible saliency map keys in outputs and metadata - if SALIENCY_KEY in inference_results.keys(): - saliency_map = inference_results[SALIENCY_KEY] - self._saliency_location = "output" - self._saliency_key = SALIENCY_KEY - elif SALIENCY_KEY in metadata.keys(): - saliency_map = metadata[SALIENCY_KEY] - self._saliency_location = "meta" - self._saliency_key = SALIENCY_KEY - elif ANOMALY_SALIENCY_KEY in metadata.keys(): - saliency_map = metadata[ANOMALY_SALIENCY_KEY] - self._saliency_location = "meta" - self._saliency_key = ANOMALY_SALIENCY_KEY - elif SEGMENTATION_SALIENCY_KEY in metadata.keys(): - saliency_map = metadata[SEGMENTATION_SALIENCY_KEY] - self._saliency_location = "meta" - self._saliency_key = SEGMENTATION_SALIENCY_KEY - else: - logging.warning("No saliency map found in model output") - saliency_map = None - - # Check all possible feature vector keys in outputs and metadata - if FEATURE_VECTOR_KEY in inference_results.keys(): - repr_vector = inference_results[FEATURE_VECTOR_KEY] - self._feature_vector_location = "output" - self._feature_vector_key = FEATURE_VECTOR_KEY - elif FEATURE_VECTOR_KEY in metadata.keys(): - repr_vector = metadata[FEATURE_VECTOR_KEY] - self._feature_vector_location = "meta" - self._feature_vector_key = FEATURE_VECTOR_KEY - else: - logging.warning("No feature vector found in model output") - repr_vector = None - else: - # If location of feature vector and saliency map are already known, we can - # use them directly - if self._saliency_location == "aux": - ( - _, - saliency_map, - repr_vector, - _, - ) = self._inference_model.postprocess_aux_outputs( - inference_results, metadata - ) - return saliency_map, repr_vector - elif self._saliency_location == "meta": - saliency_map = metadata[self._saliency_key] - elif self._saliency_location == "output": - saliency_map = inference_results[self._saliency_key] - else: - logging.warning("No saliency map found in model output") - saliency_map = None - if self._feature_vector_location == "meta": - repr_vector = metadata[self._feature_vector_key] - elif self._feature_vector_location == "output": - repr_vector = inference_results[self._feature_vector_key] - - else: - logging.warning("No feature vector found in model output") - repr_vector = None - return saliency_map, repr_vector - def infer(self, image: np.ndarray, explain: bool = False) -> Prediction: """ Run inference on an already preprocessed image. @@ -608,18 +515,13 @@ def infer(self, image: np.ndarray, explain: bool = False) -> Prediction: # Add optional explainability outputs if explain: - if not self._tiling_enabled: - saliency_map, repr_vector = self._postprocess_explain_outputs( - inference_results=inference_results, metadata=metadata - ) - else: - repr_vector = postprocessing_results.feature_vector - saliency_map = postprocessing_results.saliency_map - - prediction.feature_vector = repr_vector + if hasattr(postprocessing_results, "feature_vector"): + prediction.feature_vector = postprocessing_results.feature_vector result_medium = ResultMedium(name="saliency map", type="saliency map") - result_medium.data = saliency_map - prediction.maps = [result_medium] + result_medium.data = self._converter.convert_saliency_map( + postprocessing_results, image_shape=image.shape + ) + prediction.maps.append(result_medium) return prediction diff --git a/geti_sdk/deployment/predictions_postprocessing/results_converter/results_to_prediction_converter.py b/geti_sdk/deployment/predictions_postprocessing/results_converter/results_to_prediction_converter.py index bffd296f..6d305835 100644 --- a/geti_sdk/deployment/predictions_postprocessing/results_converter/results_to_prediction_converter.py +++ b/geti_sdk/deployment/predictions_postprocessing/results_converter/results_to_prediction_converter.py @@ -50,22 +50,26 @@ class InferenceResultsToPredictionConverter(metaclass=abc.ABCMeta): """Interface for the converter""" - @property @abc.abstractmethod - def domain(self) -> Domain: + def convert_to_prediction( + self, inference_results: NamedTuple, **kwargs + ) -> Prediction: """ - Return the domain for which the converter applies + Convert raw inference results to the Prediction format. - :return: The task domain for which the label converter applies + :param inference_results: raw predictions from inference + :return: Prediction object containing the shapes obtained from the raw predictions. """ raise NotImplementedError @abc.abstractmethod - def convert_to_prediction(self, predictions: NamedTuple, **kwargs) -> Prediction: + def convert_saliency_map( + self, inference_results: NamedTuple, **kwargs + ) -> Dict[str, np.ndarray]: """ - Convert raw predictions to Prediction format. + Extract a saliency map from inference results and return in a unified format. - :param predictions: raw predictions from inference + :param inference_results: raw predictions from inference :return: Prediction object containing the shapes obtained from the raw predictions. """ raise NotImplementedError @@ -78,8 +82,6 @@ class ClassificationToPredictionConverter(InferenceResultsToPredictionConverter) :param label_schema: LabelSchema containing the label info of the task """ - domain = Domain.CLASSIFICATION - def __init__(self, label_schema: LabelSchema): all_labels = label_schema.get_labels(include_empty=True) # add empty labels if only one non-empty label exits @@ -97,18 +99,19 @@ def __init__(self, label_schema: LabelSchema): def convert_to_prediction( self, - predictions: ClassificationResult, + inference_results: ClassificationResult, image_shape: Tuple[int, int, int], **kwargs, ) -> Prediction: # noqa: ARG003 """ - Convert ModelAPI ClassificationResult predictions to Prediction object. + Convert ModelAPI ClassificationResult inference results to Prediction object. - :param predictions: classification labels represented in ModelAPI format (label_index, label_name, confidence) + :param inference_results: classification labels represented in ModelAPI format (label_index, label_name, confidence) + :param image_shape: shape of the input image :return: Prediction object with corresponding label """ labels = [] - for label in predictions.top_labels: + for label in inference_results.top_labels: labels.append( ScoredLabel.from_label(self.labels[label[0]], float(label[-1])) ) @@ -122,6 +125,29 @@ def convert_to_prediction( ) return Prediction([annotations]) + def convert_saliency_map( + self, + inference_results: NamedTuple, + image_shape: Tuple[int, int, int], + ) -> Dict[str, np.ndarray]: + """ + Extract a saliency map from inference results and return in a unified format. + + :param inference_results: classification labels represented in ModelAPI format (label_index, label_name, confidence) + :param image_shape: shape of the input image + :return: Prediction object with corresponding label + """ + saliency_map = inference_results.saliency_map.squeeze(0) + saliency_map = cv2.resize( + np.transpose(saliency_map, (1, 2, 0)), + dsize=(image_shape[1], image_shape[0]), + interpolation=cv2.INTER_CUBIC, + ) + if len(saliency_map.shape) == 2: + saliency_map = np.expand_dims(saliency_map, axis=-1) + saliency_map = np.transpose(saliency_map, (2, 0, 1)) # shape: (N classes, h, w) + return {label.name: saliency_map[i] for i, label in enumerate(self.labels)} + class DetectionToPredictionConverter(InferenceResultsToPredictionConverter): """ @@ -131,13 +157,11 @@ class DetectionToPredictionConverter(InferenceResultsToPredictionConverter): :param configuration: optional model configuration setting """ - domain = Domain.DETECTION - def __init__( self, label_schema: LabelSchema, configuration: Optional[Dict[str, Any]] = None ): self.labels = label_schema.get_labels(include_empty=False) - self.label_map = dict(enumerate(self.labels)) + # self.label_map = dict(enumerate(self.labels)) self.use_ellipse_shapes = False self.confidence_threshold = 0.0 if configuration is not None: @@ -147,12 +171,12 @@ def __init__( self.confidence_threshold = configuration["confidence_threshold"] def convert_to_prediction( - self, predictions: DetectionResult, **kwargs + self, inference_results: DetectionResult, **kwargs ) -> Prediction: """ - Convert ModelAPI DetectionResult predictions to Prediction object. + Convert ModelAPI DetectionResult inference results to Prediction object. - :param predictions: detection represented in ModelAPI format (label, confidence, x1, y1, x2, y2). + :param inference_results: detection represented in ModelAPI format (label, confidence, x1, y1, x2, y2). _Note: - `label` can be any integer that can be mapped to `self.labels` @@ -160,7 +184,7 @@ def convert_to_prediction( - `x1`, `x2`, `y1` and `y2` are expected to be in pixel :return: Prediction object containing the boxes obtained from the prediction """ - detections = detection2array(predictions.objects) + detections = detection2array(inference_results.objects) annotations = [] if ( @@ -179,7 +203,7 @@ def convert_to_prediction( label = int(_detection[0]) confidence = _detection[1] - scored_label = ScoredLabel.from_label(self.label_map[label], confidence) + scored_label = ScoredLabel.from_label(self.labels[label], confidence) coords = _detection[2:] shape: Union[Ellipse, Rectangle] @@ -197,6 +221,29 @@ def convert_to_prediction( annotations.append(annotation) return Prediction(annotations) + def convert_saliency_map( + self, + inference_results: NamedTuple, + image_shape: Tuple[int, int, int], + ) -> Dict[str, np.ndarray]: + """ + Extract a saliency map from inference results and return in a unified format. + + :param inference_results: classification labels represented in ModelAPI format (label_index, label_name, confidence) + :param image_shape: shape of the input image + :return: Prediction object with corresponding label + """ + saliency_map = inference_results.saliency_map.squeeze(0) + saliency_map = cv2.resize( + np.transpose(saliency_map, (1, 2, 0)), + dsize=(image_shape[1], image_shape[0]), + interpolation=cv2.INTER_CUBIC, + ) + if len(saliency_map.shape) == 2: + saliency_map = np.expand_dims(saliency_map, axis=-1) + saliency_map = np.transpose(saliency_map, (2, 0, 1)) # shape: (N classes, h, w) + return {label.name: saliency_map[i] for i, label in enumerate(self.labels)} + class RotatedRectToPredictionConverter(DetectionToPredictionConverter): """ @@ -205,21 +252,19 @@ class RotatedRectToPredictionConverter(DetectionToPredictionConverter): :param label_schema: LabelSchema containing the label info of the task """ - domain = Domain.ROTATED_DETECTION - def convert_to_prediction( - self, predictions: InstanceSegmentationResult, **kwargs + self, inference_results: InstanceSegmentationResult, **kwargs ) -> Prediction: """ - Convert ModelAPI instance segmentation predictions to a rotated bounding box annotation format. + Convert ModelAPI instance segmentation inference results to a rotated bounding box annotation format. - :param predictions: segmentation represented in ModelAPI format + :param inference_results: segmentation represented in ModelAPI format :return: Prediction object containing the rotated boxes obtained from the segmentation contours :raises ValueError: if metadata is missing from the preprocess step """ annotations = [] shape: Union[RotatedRectangle, Ellipse] - for obj in predictions.segmentedObjects: + for obj in inference_results.segmentedObjects: if obj.score < self.confidence_threshold: continue if self.use_ellipse_shapes: @@ -272,8 +317,6 @@ def convert_to_prediction( class MaskToAnnotationConverter(InferenceResultsToPredictionConverter): """Converts DetectionBox Predictions ModelAPI to Prediction object.""" - domain = Domain.INSTANCE_SEGMENTATION - def __init__( self, label_schema: LabelSchema, configuration: Optional[Dict[str, Any]] = None ): @@ -287,17 +330,17 @@ def __init__( self.confidence_threshold = configuration["confidence_threshold"] def convert_to_prediction( - self, predictions: Any, **kwargs: Dict[str, Any] + self, inference_results: Any, **kwargs: Dict[str, Any] ) -> Prediction: """ - Convert predictions to Prediction object. + Convert inference results to Prediction object. - :param predictions: Raw predictions from the model. + :param inference_results: Raw inference results from the model. :return: Prediction object. """ annotations = [] shape: Union[Polygon, Ellipse] - for obj in predictions.segmentedObjects: + for obj in inference_results.segmentedObjects: if obj.score < self.confidence_threshold: continue if self.use_ellipse_shapes: @@ -347,6 +390,28 @@ def convert_to_prediction( ) return Prediction(annotations) + def convert_saliency_map( + self, + inference_results: NamedTuple, + image_shape: Tuple[int, int, int], + ) -> Dict[str, np.ndarray]: + """ + Extract a saliency map from inference results and return in a unified format. + + :param inference_results: classification labels represented in ModelAPI format (label_index, label_name, confidence) + :param image_shape: shape of the input image + :return: Prediction object with corresponding label + """ + # Model API returns a list of np.ndarray for each label + # Including `no_object` which is empty + saliency_map = np.array( + [ + smap if len(smap) > 0 else np.zeros(image_shape[:2], dtype=np.uint8) + for smap in inference_results.saliency_map + ] + ) # shape: (N classes, h, w) + return {label.name: saliency_map[i] for i, label in enumerate(self.labels)} + class SegmentationToPredictionConverter(InferenceResultsToPredictionConverter): """ @@ -355,29 +420,44 @@ class SegmentationToPredictionConverter(InferenceResultsToPredictionConverter): :param label_schema: LabelSchema containing the label info of the task """ - domain = Domain.SEGMENTATION - def __init__(self, label_schema: LabelSchema): self.labels = label_schema.get_labels(include_empty=False) # NB: index=0 is reserved for the background label self.label_map = dict(enumerate(self.labels, 1)) def convert_to_prediction( - self, predictions: ImageResultWithSoftPrediction, **kwargs # noqa: ARG002 + self, inference_results: ImageResultWithSoftPrediction, **kwargs # noqa: ARG002 ) -> Prediction: """ - Convert ModelAPI instance segmentation predictions to Prediction object. + Convert ModelAPI instance segmentation inference results to Prediction object. - :param predictions: segmentation represented in ModelAPI format + :param inference_results: segmentation represented in ModelAPI format :return: Prediction object containing the contour polygon obtained from the segmentation """ annotations = create_annotation_from_segmentation_map( - hard_prediction=predictions.resultImage, - soft_prediction=predictions.soft_prediction, + hard_prediction=inference_results.resultImage, + soft_prediction=inference_results.soft_prediction, label_map=self.label_map, ) return Prediction(annotations) + def convert_saliency_map( + self, + inference_results: NamedTuple, + image_shape: Tuple[int, int, int], + ) -> Dict[str, np.ndarray]: + """ + Extract a saliency map from inference results and return in a unified format. + + :param inference_results: classification labels represented in ModelAPI format (label_index, label_name, confidence) + :param image_shape: shape of the input image + :return: Prediction object with corresponding label + """ + saliency_map = np.transpose( + inference_results.saliency_map, (2, 0, 1) + ) # shape: (N classes, h, w) + return {label.name: saliency_map[i] for i, label in enumerate(self.labels)} + class AnomalyToPredictionConverter(InferenceResultsToPredictionConverter): """ @@ -386,8 +466,6 @@ class AnomalyToPredictionConverter(InferenceResultsToPredictionConverter): :param label_schema: LabelSchema containing the label info of the task """ - domain = Domain.ANOMALY - def __init__(self, label_schema: LabelSchema): self.labels = label_schema.get_labels(include_empty=False) self.normal_label = next( @@ -399,18 +477,18 @@ def __init__(self, label_schema: LabelSchema): self.domain = self.anomalous_label.domain def convert_to_prediction( - self, predictions: AnomalyResult, image_shape: Tuple[int], **kwargs + self, inference_results: AnomalyResult, image_shape: Tuple[int], **kwargs ) -> Prediction: # noqa: ARG002 """ - Convert ModelAPI AnomalyResult predictions to sc_sdk annotations. + Convert ModelAPI AnomalyResult inferenceresults to sc_sdk annotations. - :param predictions: anomaly result represented in ModelAPI format (same for all anomaly tasks) + :param inference_results: anomaly result represented in ModelAPI format (same for all anomaly tasks) :return: Prediction object based on the specific anomaly task: - Classification: single label (normal or anomalous). - Segmentation: contour polygon representing the segmentation. - Detection: predicted bounding boxes. """ - pred_label = predictions.pred_label + pred_label = inference_results.pred_label label = ( self.anomalous_label if pred_label in ("Anomaly", "Anomalous") @@ -419,7 +497,7 @@ def convert_to_prediction( annotations: List[Annotation] = [] if self.domain == Domain.ANOMALY_CLASSIFICATION: scored_label = ScoredLabel.from_label( - label=label, probability=float(predictions.pred_score) + label=label, probability=float(inference_results.pred_score) ) annotations = [ Annotation( @@ -429,12 +507,12 @@ def convert_to_prediction( ] elif self.domain == Domain.ANOMALY_SEGMENTATION: annotations = create_annotation_from_segmentation_map( - hard_prediction=predictions.pred_mask, - soft_prediction=predictions.anomaly_map.squeeze(), + hard_prediction=inference_results.pred_mask, + soft_prediction=inference_results.anomaly_map.squeeze(), label_map={0: self.normal_label, 1: self.anomalous_label}, ) elif self.domain == Domain.ANOMALY_DETECTION: - for box in predictions.pred_boxes: + for box in inference_results.pred_boxes: annotations.append( Annotation( shape=Rectangle( @@ -443,14 +521,14 @@ def convert_to_prediction( labels=[ ScoredLabel.from_label( label=self.anomalous_label, - probability=predictions.pred_score, + probability=inference_results.pred_score, ) ], ) ) else: raise ValueError( - f"Cannot convert predictions for task '{self.domain.name}'. Only Anomaly tasks are supported." + f"Cannot convert inference results for task '{self.domain.name}'. Only Anomaly tasks are supported." ) if not annotations: scored_label = ScoredLabel.from_label( @@ -464,6 +542,25 @@ def convert_to_prediction( ] return Prediction(annotations) + def convert_saliency_map( + self, + inference_results: NamedTuple, + image_shape: Tuple[int, int, int], + ) -> Dict[str, np.ndarray]: + """ + Extract a saliency map from inference results and return in a unified format. + + :param inference_results: classification labels represented in ModelAPI format (label_index, label_name, confidence) + :param image_shape: shape of the input image + :return: Prediction object with corresponding label + """ + # Normalizing Anomaly map + saliency_map = inference_results.anomaly_map + saliency_map -= saliency_map.min() + saliency_map = saliency_map / (saliency_map.max() + 1e-12) * 255 + saliency_map = np.round(saliency_map).astype(np.uint8) # shape: (h, w) + return {self.anomalous_label.name: saliency_map} + class ConverterFactory: """ diff --git a/geti_sdk/prediction_visualization/visualizer.py b/geti_sdk/prediction_visualization/visualizer.py index afa23887..54656aeb 100644 --- a/geti_sdk/prediction_visualization/visualizer.py +++ b/geti_sdk/prediction_visualization/visualizer.py @@ -21,6 +21,7 @@ import numpy as np from geti_sdk.data_models.annotation_scene import AnnotationScene +from geti_sdk.data_models.predictions import Prediction from geti_sdk.prediction_visualization.shape_drawer import ShapeDrawer @@ -93,6 +94,43 @@ def draw( ) return result + def explain_label( + self, + image: np.ndarray, + prediction: Prediction, + label_name: str, + opacity: float = 0.5, + ): + """ + Draw saliency map overlay on the image. + + :param image: Input image in RGB format + :param prediction: Prediction object containing saliency maps + :param label_name: Label name to be explained + :param opacity: Opacity of the saliency map overlay + :return: Output image with saliency map overlay in RGB format + """ + saliency_map = None + for pred_map in prediction.maps: + if pred_map.type == "saliency map": + saliency_map = pred_map.data + break + if saliency_map is None: + raise ValueError("Prediction does not contain saliency maps") + if label_name not in saliency_map: + raise ValueError( + f"Saliency map for label {label_name} is not found in the prediction." + ) + # Accessing the saliency map for the label + saliency_map = saliency_map[label_name] + if saliency_map.shape[:2] != image.shape[:2]: + saliency_map = cv2.resize(saliency_map, (image.shape[1], image.shape[0])) + # Visualizing the saliency map + overlay = cv2.applyColorMap(saliency_map, cv2.COLORMAP_JET) + overlay = cv2.cvtColor(overlay, cv2.COLOR_BGR2RGB) + result = cv2.addWeighted(image, 1 - opacity, overlay, opacity, 0) + return result + def show(self, image: np.ndarray) -> None: """ Show result image. From 0b950cf6bfc64764d64eba08948e77feee5085be Mon Sep 17 00:00:00 2001 From: Igor Davidyuk Date: Wed, 29 May 2024 15:53:05 +0200 Subject: [PATCH 2/7] fix tests add annotations to map visualisation Signed-off-by: Igor Davidyuk --- geti_sdk/data_models/annotation_scene.py | 8 +++-- .../results_to_prediction_converter.py | 31 +++++++++++-------- .../prediction_visualization/visualizer.py | 4 +++ 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/geti_sdk/data_models/annotation_scene.py b/geti_sdk/data_models/annotation_scene.py index df91b5f6..f36dabca 100644 --- a/geti_sdk/data_models/annotation_scene.py +++ b/geti_sdk/data_models/annotation_scene.py @@ -368,16 +368,18 @@ def map_labels( ) def filter_annotations( - self, labels: Sequence[Union[Label, ScoredLabel]] + self, labels: Sequence[Union[Label, ScoredLabel, str]] ) -> "AnnotationScene": """ Filter annotations in the scene to only include labels that are present in the provided list of labels. - :param labels: List of labels to filter the scene with + :param labels: List of labels or label names to filter the scene with :return: AnnotationScene with filtered annotations """ - label_names_to_keep = {label.name for label in labels} + label_names_to_keep = { + label if type(label) == str else label.name for label in labels + } filtered_annotations: List[Annotation] = [] for annotation in self.annotations: for label_name in annotation.label_names: diff --git a/geti_sdk/deployment/predictions_postprocessing/results_converter/results_to_prediction_converter.py b/geti_sdk/deployment/predictions_postprocessing/results_converter/results_to_prediction_converter.py index 6d305835..068f356a 100644 --- a/geti_sdk/deployment/predictions_postprocessing/results_converter/results_to_prediction_converter.py +++ b/geti_sdk/deployment/predictions_postprocessing/results_converter/results_to_prediction_converter.py @@ -65,7 +65,7 @@ def convert_to_prediction( @abc.abstractmethod def convert_saliency_map( self, inference_results: NamedTuple, **kwargs - ) -> Dict[str, np.ndarray]: + ) -> Dict[str, np.ndarray] | None: """ Extract a saliency map from inference results and return in a unified format. @@ -129,7 +129,7 @@ def convert_saliency_map( self, inference_results: NamedTuple, image_shape: Tuple[int, int, int], - ) -> Dict[str, np.ndarray]: + ) -> Dict[str, np.ndarray] | None | None: """ Extract a saliency map from inference results and return in a unified format. @@ -137,9 +137,11 @@ def convert_saliency_map( :param image_shape: shape of the input image :return: Prediction object with corresponding label """ - saliency_map = inference_results.saliency_map.squeeze(0) + saliency_map = inference_results.saliency_map + if not saliency_map: + return None saliency_map = cv2.resize( - np.transpose(saliency_map, (1, 2, 0)), + np.transpose(saliency_map.squeeze(0), (1, 2, 0)), dsize=(image_shape[1], image_shape[0]), interpolation=cv2.INTER_CUBIC, ) @@ -225,7 +227,7 @@ def convert_saliency_map( self, inference_results: NamedTuple, image_shape: Tuple[int, int, int], - ) -> Dict[str, np.ndarray]: + ) -> Dict[str, np.ndarray] | None: """ Extract a saliency map from inference results and return in a unified format. @@ -233,9 +235,11 @@ def convert_saliency_map( :param image_shape: shape of the input image :return: Prediction object with corresponding label """ - saliency_map = inference_results.saliency_map.squeeze(0) + saliency_map = inference_results.saliency_map + if not saliency_map: + return None saliency_map = cv2.resize( - np.transpose(saliency_map, (1, 2, 0)), + np.transpose(saliency_map.squeeze(0), (1, 2, 0)), dsize=(image_shape[1], image_shape[0]), interpolation=cv2.INTER_CUBIC, ) @@ -394,7 +398,7 @@ def convert_saliency_map( self, inference_results: NamedTuple, image_shape: Tuple[int, int, int], - ) -> Dict[str, np.ndarray]: + ) -> Dict[str, np.ndarray] | None: """ Extract a saliency map from inference results and return in a unified format. @@ -445,7 +449,7 @@ def convert_saliency_map( self, inference_results: NamedTuple, image_shape: Tuple[int, int, int], - ) -> Dict[str, np.ndarray]: + ) -> Dict[str, np.ndarray] | None: """ Extract a saliency map from inference results and return in a unified format. @@ -453,9 +457,10 @@ def convert_saliency_map( :param image_shape: shape of the input image :return: Prediction object with corresponding label """ - saliency_map = np.transpose( - inference_results.saliency_map, (2, 0, 1) - ) # shape: (N classes, h, w) + saliency_map = inference_results.saliency_map + if not saliency_map: + return None + saliency_map = np.transpose(saliency_map, (2, 0, 1)) # shape: (N classes, h, w) return {label.name: saliency_map[i] for i, label in enumerate(self.labels)} @@ -546,7 +551,7 @@ def convert_saliency_map( self, inference_results: NamedTuple, image_shape: Tuple[int, int, int], - ) -> Dict[str, np.ndarray]: + ) -> Dict[str, np.ndarray] | None: """ Extract a saliency map from inference results and return in a unified format. diff --git a/geti_sdk/prediction_visualization/visualizer.py b/geti_sdk/prediction_visualization/visualizer.py index 54656aeb..c0df39a4 100644 --- a/geti_sdk/prediction_visualization/visualizer.py +++ b/geti_sdk/prediction_visualization/visualizer.py @@ -100,6 +100,7 @@ def explain_label( prediction: Prediction, label_name: str, opacity: float = 0.5, + show_predictions: bool = True, ): """ Draw saliency map overlay on the image. @@ -129,6 +130,9 @@ def explain_label( overlay = cv2.applyColorMap(saliency_map, cv2.COLORMAP_JET) overlay = cv2.cvtColor(overlay, cv2.COLOR_BGR2RGB) result = cv2.addWeighted(image, 1 - opacity, overlay, opacity, 0) + if show_predictions: + filtered_prediction = prediction.filter_annotations([label_name]) + result = self.draw(result, filtered_prediction, fill_shapes=False) return result def show(self, image: np.ndarray) -> None: From 3bd00bc3913005367704b88bdd7b4428ad87a4f9 Mon Sep 17 00:00:00 2001 From: Igor Davidyuk Date: Wed, 29 May 2024 16:40:38 +0200 Subject: [PATCH 3/7] fix segmentation map order Signed-off-by: Igor Davidyuk --- .../results_converter/results_to_prediction_converter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/geti_sdk/deployment/predictions_postprocessing/results_converter/results_to_prediction_converter.py b/geti_sdk/deployment/predictions_postprocessing/results_converter/results_to_prediction_converter.py index 068f356a..4164d0ee 100644 --- a/geti_sdk/deployment/predictions_postprocessing/results_converter/results_to_prediction_converter.py +++ b/geti_sdk/deployment/predictions_postprocessing/results_converter/results_to_prediction_converter.py @@ -138,7 +138,7 @@ def convert_saliency_map( :return: Prediction object with corresponding label """ saliency_map = inference_results.saliency_map - if not saliency_map: + if len(saliency_map) == 0: return None saliency_map = cv2.resize( np.transpose(saliency_map.squeeze(0), (1, 2, 0)), @@ -236,7 +236,7 @@ def convert_saliency_map( :return: Prediction object with corresponding label """ saliency_map = inference_results.saliency_map - if not saliency_map: + if len(saliency_map) == 0: return None saliency_map = cv2.resize( np.transpose(saliency_map.squeeze(0), (1, 2, 0)), @@ -458,10 +458,10 @@ def convert_saliency_map( :return: Prediction object with corresponding label """ saliency_map = inference_results.saliency_map - if not saliency_map: + if len(saliency_map) == 0: return None saliency_map = np.transpose(saliency_map, (2, 0, 1)) # shape: (N classes, h, w) - return {label.name: saliency_map[i] for i, label in enumerate(self.labels)} + return {label.name: saliency_map[i + 1] for i, label in enumerate(self.labels)} class AnomalyToPredictionConverter(InferenceResultsToPredictionConverter): From 1b87369113cd13b07c1ddce1acb2f79bfd9b0d8a Mon Sep 17 00:00:00 2001 From: Igor Davidyuk Date: Wed, 29 May 2024 16:41:52 +0200 Subject: [PATCH 4/7] fix typo Signed-off-by: Igor Davidyuk --- geti_sdk/data_models/annotation_scene.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geti_sdk/data_models/annotation_scene.py b/geti_sdk/data_models/annotation_scene.py index f36dabca..5231f9ad 100644 --- a/geti_sdk/data_models/annotation_scene.py +++ b/geti_sdk/data_models/annotation_scene.py @@ -378,7 +378,7 @@ def filter_annotations( :return: AnnotationScene with filtered annotations """ label_names_to_keep = { - label if type(label) == str else label.name for label in labels + label if type(label) is str else label.name for label in labels } filtered_annotations: List[Annotation] = [] for annotation in self.annotations: From f9a6a34d39fed3365288c1f72c945fb97c1e538c Mon Sep 17 00:00:00 2001 From: Igor Davidyuk Date: Wed, 29 May 2024 16:48:13 +0200 Subject: [PATCH 5/7] fix typing Signed-off-by: Igor Davidyuk --- .../results_to_prediction_converter.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/geti_sdk/deployment/predictions_postprocessing/results_converter/results_to_prediction_converter.py b/geti_sdk/deployment/predictions_postprocessing/results_converter/results_to_prediction_converter.py index 4164d0ee..3baf2e08 100644 --- a/geti_sdk/deployment/predictions_postprocessing/results_converter/results_to_prediction_converter.py +++ b/geti_sdk/deployment/predictions_postprocessing/results_converter/results_to_prediction_converter.py @@ -65,7 +65,7 @@ def convert_to_prediction( @abc.abstractmethod def convert_saliency_map( self, inference_results: NamedTuple, **kwargs - ) -> Dict[str, np.ndarray] | None: + ) -> Optional[Dict[str, np.ndarray]]: """ Extract a saliency map from inference results and return in a unified format. @@ -129,7 +129,7 @@ def convert_saliency_map( self, inference_results: NamedTuple, image_shape: Tuple[int, int, int], - ) -> Dict[str, np.ndarray] | None | None: + ) -> Optional[Dict[str, np.ndarray]]: """ Extract a saliency map from inference results and return in a unified format. @@ -227,7 +227,7 @@ def convert_saliency_map( self, inference_results: NamedTuple, image_shape: Tuple[int, int, int], - ) -> Dict[str, np.ndarray] | None: + ) -> Optional[Dict[str, np.ndarray]]: """ Extract a saliency map from inference results and return in a unified format. @@ -398,7 +398,7 @@ def convert_saliency_map( self, inference_results: NamedTuple, image_shape: Tuple[int, int, int], - ) -> Dict[str, np.ndarray] | None: + ) -> Optional[Dict[str, np.ndarray]]: """ Extract a saliency map from inference results and return in a unified format. @@ -449,7 +449,7 @@ def convert_saliency_map( self, inference_results: NamedTuple, image_shape: Tuple[int, int, int], - ) -> Dict[str, np.ndarray] | None: + ) -> Optional[Dict[str, np.ndarray]]: """ Extract a saliency map from inference results and return in a unified format. @@ -551,7 +551,7 @@ def convert_saliency_map( self, inference_results: NamedTuple, image_shape: Tuple[int, int, int], - ) -> Dict[str, np.ndarray] | None: + ) -> Optional[Dict[str, np.ndarray]]: """ Extract a saliency map from inference results and return in a unified format. From 76e4c98f7381811d275f9b2dc687452a0c511d62 Mon Sep 17 00:00:00 2001 From: Igor Davidyuk Date: Thu, 30 May 2024 11:32:07 +0300 Subject: [PATCH 6/7] Update geti_sdk/prediction_visualization/visualizer.py Co-authored-by: Ludo Cornelissen --- geti_sdk/prediction_visualization/visualizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geti_sdk/prediction_visualization/visualizer.py b/geti_sdk/prediction_visualization/visualizer.py index c0df39a4..bd5396b1 100644 --- a/geti_sdk/prediction_visualization/visualizer.py +++ b/geti_sdk/prediction_visualization/visualizer.py @@ -101,7 +101,7 @@ def explain_label( label_name: str, opacity: float = 0.5, show_predictions: bool = True, - ): + ) -> np.ndarray: """ Draw saliency map overlay on the image. From 2603325edcc67292dc832e0cdc7578cfe52b68a3 Mon Sep 17 00:00:00 2001 From: Igor Davidyuk Date: Thu, 30 May 2024 10:44:16 +0200 Subject: [PATCH 7/7] address review comments Signed-off-by: Igor Davidyuk --- .../results_converter/results_to_prediction_converter.py | 1 - geti_sdk/prediction_visualization/visualizer.py | 7 ++++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/geti_sdk/deployment/predictions_postprocessing/results_converter/results_to_prediction_converter.py b/geti_sdk/deployment/predictions_postprocessing/results_converter/results_to_prediction_converter.py index 3baf2e08..5186401f 100644 --- a/geti_sdk/deployment/predictions_postprocessing/results_converter/results_to_prediction_converter.py +++ b/geti_sdk/deployment/predictions_postprocessing/results_converter/results_to_prediction_converter.py @@ -163,7 +163,6 @@ def __init__( self, label_schema: LabelSchema, configuration: Optional[Dict[str, Any]] = None ): self.labels = label_schema.get_labels(include_empty=False) - # self.label_map = dict(enumerate(self.labels)) self.use_ellipse_shapes = False self.confidence_threshold = 0.0 if configuration is not None: diff --git a/geti_sdk/prediction_visualization/visualizer.py b/geti_sdk/prediction_visualization/visualizer.py index bd5396b1..c69035f2 100644 --- a/geti_sdk/prediction_visualization/visualizer.py +++ b/geti_sdk/prediction_visualization/visualizer.py @@ -109,6 +109,7 @@ def explain_label( :param prediction: Prediction object containing saliency maps :param label_name: Label name to be explained :param opacity: Opacity of the saliency map overlay + :param show_predictions: Show predictions for the label on the output image :return: Output image with saliency map overlay in RGB format """ saliency_map = None @@ -125,7 +126,11 @@ def explain_label( # Accessing the saliency map for the label saliency_map = saliency_map[label_name] if saliency_map.shape[:2] != image.shape[:2]: - saliency_map = cv2.resize(saliency_map, (image.shape[1], image.shape[0])) + saliency_map = cv2.resize( + saliency_map, + (image.shape[1], image.shape[0]), + interpolation=cv2.INTER_CUBIC, + ) # Visualizing the saliency map overlay = cv2.applyColorMap(saliency_map, cv2.COLORMAP_JET) overlay = cv2.cvtColor(overlay, cv2.COLOR_BGR2RGB)