Enhance detection & instance segmentation experiment

src/otx/algorithms/detection/adapters/openvino/task.py

@@ -619,6 +619,7 @@ def evaluate(
                 f"Requested to use {evaluation_metric} metric, but parameter is ignored. Use F-measure instead."
             )
         output_resultset.performance = MetricsHelper.compute_f_measure(output_resultset).get_performance()
+        logger.info(f"F-measure after evaluation: {output_resultset.performance}")
         logger.info("OpenVINO metric evaluation completed")
 
     def deploy(self, output_model: ModelEntity) -> None:

src/otx/algorithms/detection/task.py

@@ -446,8 +446,8 @@ def evaluate(
                 f"Requested to use {evaluation_metric} metric, " "but parameter is ignored. Use F-measure instead."
             )
         metric = MetricsHelper.compute_f_measure(output_resultset)
-        logger.info(f"F-measure after evaluation: {metric.f_measure.value}")
         output_resultset.performance = metric.get_performance()
+        logger.info(f"F-measure after evaluation: {output_resultset.performance}")
         logger.info("Evaluation completed")
 
     def _add_predictions_to_dataset(

src/otx/api/usecases/evaluation/f_measure.py

@@ -19,7 +19,7 @@
     LineChartInfo,
     LineMetricsGroup,
     MetricsGroup,
-    Performance,
+    MultiScorePerformance,
     ScoreMetric,
     TextChartInfo,
     TextMetricsGroup,
@@ -205,6 +205,7 @@ class _AggregatedResults:
         - all_classes_f_measure_curve
         - best_f_measure
         - best_threshold
+        - best_f_measure_metrics
 
     Args:
         classes (List[str]): List of classes.
@@ -217,6 +218,7 @@ def __init__(self, classes: List[str]):
         self.all_classes_f_measure_curve: List[float] = []
         self.best_f_measure: float = 0.0
         self.best_threshold: float = 0.0
+        self.best_f_measure_metrics: _Metrics = _Metrics(0.0, 0.0, 0.0)
 
 
 class _OverallResults:
@@ -364,6 +366,7 @@ def get_results_per_confidence(
             if all_classes_f_measure > 0.0 and all_classes_f_measure >= result.best_f_measure:
                 result.best_f_measure = all_classes_f_measure
                 result.best_threshold = confidence_threshold
+                result.best_f_measure_metrics = result_point[ALL_CLASSES_NAME]
         return result
 
     def get_results_per_nms(
@@ -418,6 +421,7 @@ def get_results_per_nms(
             if all_classes_f_measure > 0.0 and all_classes_f_measure >= result.best_f_measure:
                 result.best_f_measure = all_classes_f_measure
                 result.best_threshold = nms_threshold
+                result.best_f_measure_metrics = result_point[ALL_CLASSES_NAME]
         return result
 
     def evaluate_classes(
@@ -693,6 +697,8 @@ def __init__(
             self.f_measure_per_label[label] = ScoreMetric(
                 name=label.name, value=result.best_f_measure_per_class[label.name]
             )
+        self._precision = ScoreMetric(name="Precision", value=result.per_confidence.best_f_measure_metrics.precision)
+        self._recall = ScoreMetric(name="Recall", value=result.per_confidence.best_f_measure_metrics.recall)
 
         self._f_measure_per_confidence: Optional[CurveMetric] = None
         self._best_confidence_threshold: Optional[ScoreMetric] = None
@@ -752,13 +758,12 @@ def best_nms_threshold(self) -> Optional[ScoreMetric]:
         """Returns the best NMS threshold as ScoreMetric if exists."""
         return self._best_nms_threshold
 
-    def get_performance(self) -> Performance:
+    def get_performance(self) -> MultiScorePerformance:
         """Returns the performance which consists of the F-Measure score and the dashboard metrics.
 
         Returns:
-            Performance: Performance object containing the F-Measure score and the dashboard metrics.
+            MultiScorePerformance: MultiScorePerformance object containing the F-Measure scores and the dashboard metrics.
         """
-        score = self.f_measure
         dashboard_metrics: List[MetricsGroup] = []
         dashboard_metrics.append(
             BarMetricsGroup(
@@ -813,7 +818,11 @@ def get_performance(self) -> Performance:
                     ),
                 )
             )
-        return Performance(score=score, dashboard_metrics=dashboard_metrics)
+        return MultiScorePerformance(
+            primary_score=self.f_measure,
+            additional_scores=[self._precision, self._recall],
+            dashboard_metrics=dashboard_metrics,
+        )
 
     @staticmethod
     def __get_boxes_from_dataset_as_list(

src/otx/cli/tools/eval.py

@@ -153,10 +153,12 @@ def main():
     )
     task.evaluate(resultset)
     assert resultset.performance is not None
-    print(resultset.performance)
 
     output_path = Path(args.output) if args.output else config_manager.output_path
     performance = {resultset.performance.score.name: resultset.performance.score.value}
+    if hasattr(resultset.performance, "additional_scores"):
+        for metric in resultset.performance.additional_scores:
+            performance[metric.name] = metric.value
     if hasattr(task, "avg_time_per_image"):
         performance["avg_time_per_image"] = task.avg_time_per_image
     with open(output_path / "performance.json", "w", encoding="UTF-8") as write_file:

src/otx/core/data/adapter/base_dataset_adapter.py

@@ -39,6 +39,7 @@
 from otx.api.entities.media import IMediaEntity
 from otx.api.entities.model_template import TaskType
 from otx.api.entities.scored_label import ScoredLabel
+from otx.api.entities.shapes.ellipse import Ellipse
 from otx.api.entities.shapes.polygon import Point, Polygon
 from otx.api.entities.shapes.rectangle import Rectangle
 from otx.api.entities.subset import Subset
@@ -350,6 +351,21 @@ def _get_polygon_entity(
             labels=[ScoredLabel(label=self.label_entities[annotation.label])],
         )
 
+    def _get_ellipse_entity(
+        self, annotation: DatumAnnotation, width: int, height: int, num_polygons: int = -1
+    ) -> Annotation:
+        """Get ellipse entity."""
+        ellipse = Ellipse(
+            annotation.x1 / (width - 1),
+            annotation.y1 / (height - 1),
+            annotation.x2 / (width - 1),
+            annotation.y2 / (height - 1),
+        )
+        return Annotation(
+            ellipse,
+            labels=[ScoredLabel(label=self.label_entities[annotation.label])],
+        )
+
     def _get_mask_entity(self, annotation: DatumAnnotation) -> Annotation:
         """Get mask entity."""
         mask = Image(data=annotation.image, size=annotation.image.shape)

src/otx/core/data/adapter/detection_dataset_adapter.py

@@ -37,14 +37,15 @@ def get_otx_dataset(self) -> DatasetEntity:
                     assert isinstance(image, Image)
                     shapes = []
                     for ann in datumaro_item.annotations:
-                        if (
-                            self.task_type in (TaskType.INSTANCE_SEGMENTATION, TaskType.ROTATED_DETECTION)
-                            and ann.type == DatumAnnotationType.polygon
-                        ):
-                            if self._is_normal_polygon(ann):
+                        if self.task_type in (TaskType.INSTANCE_SEGMENTATION, TaskType.ROTATED_DETECTION):
+                            if ann.type == DatumAnnotationType.polygon and self._is_normal_polygon(ann):
                                 shapes.append(self._get_polygon_entity(ann, image.width, image.height))
-                        if self.task_type is TaskType.DETECTION and ann.type == DatumAnnotationType.bbox:
-                            if self._is_normal_bbox(ann.points[0], ann.points[1], ann.points[2], ann.points[3]):
+                            elif ann.type == DatumAnnotationType.ellipse:
+                                shapes.append(self._get_ellipse_entity(ann, image.width, image.height))
+                        elif self.task_type is TaskType.DETECTION:
+                            if ann.type == DatumAnnotationType.bbox and self._is_normal_bbox(
+                                ann.points[0], ann.points[1], ann.points[2], ann.points[3]
+                            ):
                                 shapes.append(self._get_normalized_bbox_entity(ann, image.width, image.height))
 
                         if ann.label not in used_labels:

tools/experiment.py

@@ -6,6 +6,7 @@
 import argparse
 import csv
 import dataclasses
+import gc
 import json
 import os
 import re
@@ -701,6 +702,8 @@ def run_command_list(self, dryrun: bool = False):
 
             self._previous_cmd_entry.append(command[1])
 
+            gc.collect()
+
         if not dryrun:
             organize_exp_result(self._workspace, self._command_var)