diff --git a/src/otx/algorithms/detection/adapters/openvino/task.py b/src/otx/algorithms/detection/adapters/openvino/task.py index eedf97f8b48..c91d14e6430 100644 --- a/src/otx/algorithms/detection/adapters/openvino/task.py +++ b/src/otx/algorithms/detection/adapters/openvino/task.py @@ -619,6 +619,7 @@ def evaluate( f"Requested to use {evaluation_metric} metric, but parameter is ignored. Use F-measure instead." ) output_resultset.performance = MetricsHelper.compute_f_measure(output_resultset).get_performance() + logger.info(f"F-measure after evaluation: {output_resultset.performance}") logger.info("OpenVINO metric evaluation completed") def deploy(self, output_model: ModelEntity) -> None: diff --git a/src/otx/algorithms/detection/task.py b/src/otx/algorithms/detection/task.py index 84ec50697f9..78af633e2a1 100644 --- a/src/otx/algorithms/detection/task.py +++ b/src/otx/algorithms/detection/task.py @@ -446,8 +446,8 @@ def evaluate( f"Requested to use {evaluation_metric} metric, " "but parameter is ignored. Use F-measure instead." ) metric = MetricsHelper.compute_f_measure(output_resultset) - logger.info(f"F-measure after evaluation: {metric.f_measure.value}") output_resultset.performance = metric.get_performance() + logger.info(f"F-measure after evaluation: {output_resultset.performance}") logger.info("Evaluation completed") def _add_predictions_to_dataset( diff --git a/src/otx/api/usecases/evaluation/f_measure.py b/src/otx/api/usecases/evaluation/f_measure.py index cc845ef7609..4837fcf193a 100644 --- a/src/otx/api/usecases/evaluation/f_measure.py +++ b/src/otx/api/usecases/evaluation/f_measure.py @@ -19,7 +19,7 @@ LineChartInfo, LineMetricsGroup, MetricsGroup, - Performance, + MultiScorePerformance, ScoreMetric, TextChartInfo, TextMetricsGroup, @@ -205,6 +205,7 @@ class _AggregatedResults: - all_classes_f_measure_curve - best_f_measure - best_threshold + - best_f_measure_metrics Args: classes (List[str]): List of classes. @@ -217,6 +218,7 @@ def __init__(self, classes: List[str]): self.all_classes_f_measure_curve: List[float] = [] self.best_f_measure: float = 0.0 self.best_threshold: float = 0.0 + self.best_f_measure_metrics: _Metrics = _Metrics(0.0, 0.0, 0.0) class _OverallResults: @@ -364,6 +366,7 @@ def get_results_per_confidence( if all_classes_f_measure > 0.0 and all_classes_f_measure >= result.best_f_measure: result.best_f_measure = all_classes_f_measure result.best_threshold = confidence_threshold + result.best_f_measure_metrics = result_point[ALL_CLASSES_NAME] return result def get_results_per_nms( @@ -418,6 +421,7 @@ def get_results_per_nms( if all_classes_f_measure > 0.0 and all_classes_f_measure >= result.best_f_measure: result.best_f_measure = all_classes_f_measure result.best_threshold = nms_threshold + result.best_f_measure_metrics = result_point[ALL_CLASSES_NAME] return result def evaluate_classes( @@ -693,6 +697,8 @@ def __init__( self.f_measure_per_label[label] = ScoreMetric( name=label.name, value=result.best_f_measure_per_class[label.name] ) + self._precision = ScoreMetric(name="Precision", value=result.per_confidence.best_f_measure_metrics.precision) + self._recall = ScoreMetric(name="Recall", value=result.per_confidence.best_f_measure_metrics.recall) self._f_measure_per_confidence: Optional[CurveMetric] = None self._best_confidence_threshold: Optional[ScoreMetric] = None @@ -752,13 +758,12 @@ def best_nms_threshold(self) -> Optional[ScoreMetric]: """Returns the best NMS threshold as ScoreMetric if exists.""" return self._best_nms_threshold - def get_performance(self) -> Performance: + def get_performance(self) -> MultiScorePerformance: """Returns the performance which consists of the F-Measure score and the dashboard metrics. Returns: - Performance: Performance object containing the F-Measure score and the dashboard metrics. + MultiScorePerformance: MultiScorePerformance object containing the F-Measure scores and the dashboard metrics. """ - score = self.f_measure dashboard_metrics: List[MetricsGroup] = [] dashboard_metrics.append( BarMetricsGroup( @@ -813,7 +818,11 @@ def get_performance(self) -> Performance: ), ) ) - return Performance(score=score, dashboard_metrics=dashboard_metrics) + return MultiScorePerformance( + primary_score=self.f_measure, + additional_scores=[self._precision, self._recall], + dashboard_metrics=dashboard_metrics, + ) @staticmethod def __get_boxes_from_dataset_as_list( diff --git a/src/otx/cli/tools/eval.py b/src/otx/cli/tools/eval.py index f210ba4a395..f609f5d3ded 100644 --- a/src/otx/cli/tools/eval.py +++ b/src/otx/cli/tools/eval.py @@ -153,10 +153,12 @@ def main(): ) task.evaluate(resultset) assert resultset.performance is not None - print(resultset.performance) output_path = Path(args.output) if args.output else config_manager.output_path performance = {resultset.performance.score.name: resultset.performance.score.value} + if hasattr(resultset.performance, "additional_scores"): + for metric in resultset.performance.additional_scores: + performance[metric.name] = metric.value if hasattr(task, "avg_time_per_image"): performance["avg_time_per_image"] = task.avg_time_per_image with open(output_path / "performance.json", "w", encoding="UTF-8") as write_file: diff --git a/src/otx/core/data/adapter/base_dataset_adapter.py b/src/otx/core/data/adapter/base_dataset_adapter.py index 51b62a0cede..52195de4d0f 100644 --- a/src/otx/core/data/adapter/base_dataset_adapter.py +++ b/src/otx/core/data/adapter/base_dataset_adapter.py @@ -39,6 +39,7 @@ from otx.api.entities.media import IMediaEntity from otx.api.entities.model_template import TaskType from otx.api.entities.scored_label import ScoredLabel +from otx.api.entities.shapes.ellipse import Ellipse from otx.api.entities.shapes.polygon import Point, Polygon from otx.api.entities.shapes.rectangle import Rectangle from otx.api.entities.subset import Subset @@ -350,6 +351,21 @@ def _get_polygon_entity( labels=[ScoredLabel(label=self.label_entities[annotation.label])], ) + def _get_ellipse_entity( + self, annotation: DatumAnnotation, width: int, height: int, num_polygons: int = -1 + ) -> Annotation: + """Get ellipse entity.""" + ellipse = Ellipse( + annotation.x1 / (width - 1), + annotation.y1 / (height - 1), + annotation.x2 / (width - 1), + annotation.y2 / (height - 1), + ) + return Annotation( + ellipse, + labels=[ScoredLabel(label=self.label_entities[annotation.label])], + ) + def _get_mask_entity(self, annotation: DatumAnnotation) -> Annotation: """Get mask entity.""" mask = Image(data=annotation.image, size=annotation.image.shape) diff --git a/src/otx/core/data/adapter/detection_dataset_adapter.py b/src/otx/core/data/adapter/detection_dataset_adapter.py index a6ce1b2bce5..612ab303f23 100644 --- a/src/otx/core/data/adapter/detection_dataset_adapter.py +++ b/src/otx/core/data/adapter/detection_dataset_adapter.py @@ -37,14 +37,15 @@ def get_otx_dataset(self) -> DatasetEntity: assert isinstance(image, Image) shapes = [] for ann in datumaro_item.annotations: - if ( - self.task_type in (TaskType.INSTANCE_SEGMENTATION, TaskType.ROTATED_DETECTION) - and ann.type == DatumAnnotationType.polygon - ): - if self._is_normal_polygon(ann): + if self.task_type in (TaskType.INSTANCE_SEGMENTATION, TaskType.ROTATED_DETECTION): + if ann.type == DatumAnnotationType.polygon and self._is_normal_polygon(ann): shapes.append(self._get_polygon_entity(ann, image.width, image.height)) - if self.task_type is TaskType.DETECTION and ann.type == DatumAnnotationType.bbox: - if self._is_normal_bbox(ann.points[0], ann.points[1], ann.points[2], ann.points[3]): + elif ann.type == DatumAnnotationType.ellipse: + shapes.append(self._get_ellipse_entity(ann, image.width, image.height)) + elif self.task_type is TaskType.DETECTION: + if ann.type == DatumAnnotationType.bbox and self._is_normal_bbox( + ann.points[0], ann.points[1], ann.points[2], ann.points[3] + ): shapes.append(self._get_normalized_bbox_entity(ann, image.width, image.height)) if ann.label not in used_labels: diff --git a/tools/experiment.py b/tools/experiment.py index 891c4e75aaa..6d9a271e547 100644 --- a/tools/experiment.py +++ b/tools/experiment.py @@ -6,6 +6,7 @@ import argparse import csv import dataclasses +import gc import json import os import re @@ -701,6 +702,8 @@ def run_command_list(self, dryrun: bool = False): self._previous_cmd_entry.append(command[1]) + gc.collect() + if not dryrun: organize_exp_result(self._workspace, self._command_var)