diff --git a/CHANGELOG.md b/CHANGELOG.md index 962dcca651e..2dc2642083d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ All notable changes to this project will be documented in this file. - Classification task refactoring () - Extend OTX explain CLI () - Segmentation task refactoring () +- Action task refactoring () ### Bug fixes diff --git a/otx/algorithms/action/adapters/mmaction/__init__.py b/otx/algorithms/action/adapters/mmaction/__init__.py index d7cbbf60aed..79b7026756b 100644 --- a/otx/algorithms/action/adapters/mmaction/__init__.py +++ b/otx/algorithms/action/adapters/mmaction/__init__.py @@ -5,8 +5,8 @@ from .data import OTXActionClsDataset, OTXActionDetDataset from .models import register_action_backbones -from .utils import Exporter, patch_config, set_data_classes +from .utils import Exporter -__all__ = ["OTXActionClsDataset", "OTXActionDetDataset", "patch_config", "set_data_classes", "Exporter"] +__all__ = ["OTXActionClsDataset", "OTXActionDetDataset", "Exporter"] register_action_backbones() diff --git a/otx/algorithms/action/adapters/mmaction/data/cls_dataset.py b/otx/algorithms/action/adapters/mmaction/data/cls_dataset.py index 3de07fdb244..903dad76036 100644 --- a/otx/algorithms/action/adapters/mmaction/data/cls_dataset.py +++ b/otx/algorithms/action/adapters/mmaction/data/cls_dataset.py @@ -104,6 +104,7 @@ def __init__( ): self.otx_dataset = otx_dataset self.labels = labels + self.CLASSES = [label.name for label in labels] self.test_mode = test_mode self.modality = modality diff --git a/otx/algorithms/action/adapters/mmaction/data/det_dataset.py b/otx/algorithms/action/adapters/mmaction/data/det_dataset.py index cbb8a509bc4..7ee87e89f39 100644 --- a/otx/algorithms/action/adapters/mmaction/data/det_dataset.py +++ b/otx/algorithms/action/adapters/mmaction/data/det_dataset.py @@ -219,6 +219,7 @@ def __init__( ): self.otx_dataset = otx_dataset self.labels = labels + self.CLASSES = [label.name for label in labels] self.test_mode = test_mode self.modality = modality self._FPS = fps diff --git a/otx/algorithms/action/adapters/mmaction/task.py b/otx/algorithms/action/adapters/mmaction/task.py new file mode 100644 index 00000000000..9754ec4f97d --- /dev/null +++ b/otx/algorithms/action/adapters/mmaction/task.py @@ -0,0 +1,526 @@ +"""Task of OTX Video Recognition using mmaction training backend.""" + +# Copyright (C) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. + +import glob +import os +import time +from copy import deepcopy +from typing import Optional, Union + +import torch +from mmaction import __version__ +from mmaction.apis import train_model +from mmaction.datasets import build_dataloader, build_dataset +from mmaction.models import build_model as build_videomodel +from mmaction.utils import collect_env +from mmcv.runner import CheckpointLoader, load_state_dict, wrap_fp16_model +from mmcv.utils import Config, ConfigDict, ProgressBar, get_git_hash + +from otx.algorithms.action.adapters.mmaction import ( + Exporter, +) +from otx.algorithms.action.task import OTXActionTask +from otx.algorithms.common.adapters.mmcv.utils import ( + build_data_parallel, + get_configs_by_pairs, + patch_adaptive_interval_training, + patch_data_pipeline, + patch_early_stopping, + patch_from_hyperparams, + patch_persistent_workers, +) +from otx.algorithms.common.adapters.mmcv.utils.config_utils import ( + MPAConfig, + update_or_add_custom_hook, +) +from otx.algorithms.common.utils import set_random_seed +from otx.algorithms.common.utils.data import get_dataset +from otx.algorithms.common.utils.logger import get_logger +from otx.api.entities.datasets import DatasetEntity +from otx.api.entities.inference_parameters import InferenceParameters +from otx.api.entities.model import ModelPrecision +from otx.api.entities.model_template import TaskType +from otx.api.entities.subset import Subset +from otx.api.entities.task_environment import TaskEnvironment +from otx.core.data import caching + +logger = get_logger() + +# TODO Remove unnecessary pylint disable +# pylint: disable=too-many-lines + + +class MMActionTask(OTXActionTask): + """Task class for OTX action using mmaction training backend.""" + + # pylint: disable=too-many-instance-attributes + def __init__(self, task_environment: TaskEnvironment, output_path: Optional[str] = None): + super().__init__(task_environment, output_path) + self._data_cfg: Optional[Config] = None + self._recipe_cfg: Optional[Config] = None + + # pylint: disable=too-many-locals, too-many-branches, too-many-statements + def _init_task(self, export: bool = False): # noqa + """Initialize task.""" + + self._recipe_cfg = MPAConfig.fromfile(os.path.join(self._model_dir, "model.py")) + self._recipe_cfg.domain = self._task_type.domain + self._config = self._recipe_cfg + + set_random_seed(self._recipe_cfg.get("seed", 5), logger, self._recipe_cfg.get("deterministic", False)) + + # Belows may go to the configure function + patch_data_pipeline(self._recipe_cfg, self.data_pipeline_path) + + if not export: + patch_from_hyperparams(self._recipe_cfg, self._hyperparams) + self._recipe_cfg.total_epochs = self._recipe_cfg.runner.max_epochs + + if "custom_hooks" in self.override_configs: + override_custom_hooks = self.override_configs.pop("custom_hooks") + for override_custom_hook in override_custom_hooks: + update_or_add_custom_hook(self._recipe_cfg, ConfigDict(override_custom_hook)) + if len(self.override_configs) > 0: + logger.info(f"before override configs merging = {self._recipe_cfg}") + self._recipe_cfg.merge_from_dict(self.override_configs) + logger.info(f"after override configs merging = {self._recipe_cfg}") + + # add Cancel training hook + update_or_add_custom_hook( + self._recipe_cfg, + ConfigDict(type="CancelInterfaceHook", init_callback=self.on_hook_initialized), + ) + if self._time_monitor is not None: + update_or_add_custom_hook( + self._recipe_cfg, + ConfigDict( + type="OTXProgressHook", + time_monitor=self._time_monitor, + verbose=True, + priority=71, + ), + ) + self._recipe_cfg.log_config.hooks.append({"type": "OTXLoggerHook", "curves": self._learning_curves}) + + # Update recipe with caching modules + self._update_caching_modules(self._recipe_cfg.data) + + logger.info("initialized.") + + def build_model( + self, + cfg: Config, + fp16: bool = False, + **kwargs, + ) -> torch.nn.Module: + """Build model from model_builder.""" + model_builder = getattr(self, "model_builder", build_videomodel) + model = model_builder(cfg.model, **kwargs) + ckpt = CheckpointLoader.load_checkpoint(cfg.load_from, map_location="cpu") + if "model" in ckpt: + ckpt = ckpt["model"] + if "state_dict" in ckpt: + ckpt = ckpt["state_dict"] + load_state_dict(model, ckpt) + if fp16: + wrap_fp16_model(model) + return model + + # pylint: disable=too-many-arguments + def configure( + self, + training=True, + subset="train", + ir_options=None, + ): + """Patch mmcv configs for OTX action settings.""" + + # deepcopy all configs to make sure + # changes under MPA and below does not take an effect to OTX for clear distinction + recipe_cfg = deepcopy(self._recipe_cfg) + data_cfg = deepcopy(self._data_cfg) + assert recipe_cfg is not None, "'recipe_cfg' is not initialized." + + recipe_cfg.work_dir = self._output_path + recipe_cfg.resume = self._resume + recipe_cfg.distributed = False + recipe_cfg.omnisource = False + + if data_cfg is not None: + recipe_cfg.merge_from_dict(data_cfg) + + if self._task_type == TaskType.ACTION_CLASSIFICATION: + _dataset_type = "OTXActionClsDataset" + else: + _dataset_type = "OTXActionDetDataset" + for subset in ("train", "val", "test", "unlabeled"): + _cfg = recipe_cfg.data.get(subset, None) + if not _cfg: + continue + _cfg.type = _dataset_type + while "dataset" in _cfg: + _cfg = _cfg.dataset + _cfg.labels = self._labels + + if self._task_type == TaskType.ACTION_CLASSIFICATION: + recipe_cfg.model["cls_head"].num_classes = len(self._labels) + elif self._task_type == TaskType.ACTION_DETECTION: + recipe_cfg.model["roi_head"]["bbox_head"].num_classes = len(self._labels) + 1 + if len(self._labels) < 5: + recipe_cfg.model["roi_head"]["bbox_head"]["topk"] = len(self._labels) - 1 + + recipe_cfg.data.videos_per_gpu = recipe_cfg.data.pop("samples_per_gpu", None) + + patch_adaptive_interval_training(recipe_cfg) + patch_early_stopping(recipe_cfg) + patch_persistent_workers(recipe_cfg) + + if self._model_ckpt is not None: + recipe_cfg.load_from = self._model_ckpt + + self._config = recipe_cfg + return recipe_cfg + + # pylint: disable=too-many-branches, too-many-statements + def _train_model( + self, + dataset: DatasetEntity, + ): + """Train function in MMActionTask.""" + logger.info("init data cfg.") + self._data_cfg = ConfigDict(data=ConfigDict()) + + for cfg_key, subset in zip( + ["train", "val", "unlabeled"], + [Subset.TRAINING, Subset.VALIDATION, Subset.UNLABELED], + ): + subset = get_dataset(dataset, subset) + if subset and self._data_cfg is not None: + self._data_cfg.data[cfg_key] = ConfigDict( + otx_dataset=subset, + labels=self._labels, + ) + + self._is_training = True + + self._init_task() + + cfg = self.configure(True, "train", None) + logger.info("train!") + + timestamp = time.strftime("%Y%m%d_%H%M%S", time.localtime()) + + # Environment + logger.info(f"cfg.gpu_ids = {cfg.gpu_ids}, distributed = {cfg.distributed}") + env_info_dict = collect_env() + env_info = "\n".join([(f"{k}: {v}") for k, v in env_info_dict.items()]) + dash_line = "-" * 60 + "\n" + logger.info(f"Environment info:\n{dash_line}{env_info}\n{dash_line}") + + # Data + datasets = [build_dataset(cfg.data.train)] + + # FIXME: Currently action do not support multi batch evaluation. This will be fixed + if "val" in cfg.data: + cfg.data.val_dataloader["videos_per_gpu"] = 1 + + # Target classes + if "task_adapt" in cfg: + target_classes = cfg.task_adapt.get("final", []) + else: + target_classes = datasets[0].CLASSES + + # Metadata + meta = dict() + meta["env_info"] = env_info + meta["seed"] = cfg.seed + meta["exp_name"] = cfg.work_dir + if cfg.checkpoint_config is not None: + cfg.checkpoint_config.meta = dict( + mmaction2_version=__version__ + get_git_hash()[:7], + CLASSES=target_classes, + ) + + # Model + model = self.build_model(cfg, fp16=cfg.get("fp16", False)) + model.train() + model.CLASSES = target_classes + + if cfg.distributed: + torch.nn.SyncBatchNorm.convert_sync_batchnorm(model) + + validate = bool(cfg.data.get("val", None)) + train_model( + model, + datasets, + cfg, + distributed=cfg.distributed, + validate=validate, + timestamp=timestamp, + meta=meta, + ) + + # Save outputs + output_ckpt_path = os.path.join(cfg.work_dir, "latest.pth") + best_ckpt_path = glob.glob(os.path.join(cfg.work_dir, "best_*.pth")) + if best_ckpt_path: + output_ckpt_path = best_ckpt_path[0] + return dict( + final_ckpt=output_ckpt_path, + ) + + def _infer_model( + self, + dataset: DatasetEntity, + inference_parameters: Optional[InferenceParameters] = None, + ): + """Main infer function.""" + self._data_cfg = ConfigDict( + data=ConfigDict( + train=ConfigDict( + otx_dataset=None, + labels=self._labels, + ), + test=ConfigDict( + otx_dataset=dataset, + labels=self._labels, + ), + ) + ) + + dump_features = False + dump_saliency_map = False + + self._init_task() + + cfg = self.configure(False, "test", None) + logger.info("infer!") + + videos_per_gpu = cfg.data.test_dataloader.get("videos_per_gpu", 1) + + # Data loader + mm_dataset = build_dataset(cfg.data.test) + dataloader = build_dataloader( + mm_dataset, + videos_per_gpu=videos_per_gpu, + workers_per_gpu=cfg.data.test_dataloader.get("workers_per_gpu", 0), + num_gpus=len(cfg.gpu_ids), + dist=cfg.distributed, + seed=cfg.get("seed", None), + shuffle=False, + ) + + # Target classes + if "task_adapt" in cfg: + target_classes = cfg.task_adapt.final + if len(target_classes) < 1: + raise KeyError( + f"target_classes={target_classes} is empty check the metadata from model ckpt or recipe " + "configuration" + ) + else: + target_classes = mm_dataset.CLASSES + + # Model + model = self.build_model(cfg, fp16=cfg.get("fp16", False)) + model.CLASSES = target_classes + model.eval() + model = build_data_parallel(model, cfg, distributed=False) + + # InferenceProgressCallback (Time Monitor enable into Infer task) + time_monitor = None + if cfg.get("custom_hooks", None): + time_monitor = [hook.time_monitor for hook in cfg.custom_hooks if hook.type == "OTXProgressHook"] + time_monitor = time_monitor[0] if time_monitor else None + if time_monitor is not None: + + # pylint: disable=unused-argument + def pre_hook(module, inp): + time_monitor.on_test_batch_begin(None, None) + + def hook(module, inp, outp): + time_monitor.on_test_batch_end(None, None) + + model.register_forward_pre_hook(pre_hook) + model.register_forward_hook(hook) + + eval_predictions = [] + feature_vectors = [] + saliency_maps = [] + + def dump_features_hook(): + raise NotImplementedError("get_feature_vector function for mmaction is not implemented") + + # pylint: disable=unused-argument + def dummy_dump_features_hook(model, inp, out): + feature_vectors.append(None) + + def dump_saliency_hook(): + raise NotImplementedError("get_saliency_map for mmaction is not implemented") + + # pylint: disable=unused-argument + def dummy_dump_saliency_hook(model, inp, out): + saliency_maps.append(None) + + feature_vector_hook = dump_features_hook if dump_features else dummy_dump_features_hook + saliency_map_hook = dump_saliency_hook if dump_saliency_map else dummy_dump_saliency_hook + + prog_bar = ProgressBar(len(dataloader)) + with model.module.backbone.register_forward_hook(feature_vector_hook): + with model.module.backbone.register_forward_hook(saliency_map_hook): + for data in dataloader: + with torch.no_grad(): + result = model(return_loss=False, **data) + eval_predictions.extend(result) + for _ in range(len(data)): + prog_bar.update() + prog_bar.file.write("\n") + + for key in ["interval", "tmpdir", "start", "gpu_collect", "save_best", "rule", "dynamic_intervals"]: + cfg.evaluation.pop(key, None) + + metric = None + metric_name = self._recipe_cfg.evaluation.final_metric + if inference_parameters: + if inference_parameters.is_evaluation: + metric = mm_dataset.evaluate(eval_predictions, **self._recipe_cfg.evaluation)[metric_name] + + assert len(eval_predictions) == len(feature_vectors), f"{len(eval_predictions)} != {len(feature_vectors)}" + assert len(eval_predictions) == len(saliency_maps), f"{len(eval_predictions)} != {len(saliency_maps)}" + predictions = zip(eval_predictions, feature_vectors, saliency_maps) + + return predictions, metric + + def _export_model(self, precision: ModelPrecision, dump_features: bool = True): + """Main export function.""" + self._init_task(export=True) + + cfg = self.configure(False, "test", None) + deploy_cfg = self._init_deploy_cfg() + + state_dict = torch.load(self._model_ckpt) + if "model" in state_dict.keys(): + state_dict = state_dict["model"] + + self._precision[0] = precision + half_precision = precision == ModelPrecision.FP16 + + exporter = Exporter(cfg, state_dict, deploy_cfg, f"{self._output_path}/openvino", half_precision) + exporter.export() + bin_file = [f for f in os.listdir(self._output_path) if f.endswith(".bin")][0] + xml_file = [f for f in os.listdir(self._output_path) if f.endswith(".xml")][0] + onnx_file = [f for f in os.listdir(self._output_path) if f.endswith(".onnx")][0] + results = { + "outputs": { + "bin": os.path.join(self._output_path, bin_file), + "xml": os.path.join(self._output_path, xml_file), + "onnx": os.path.join(self._output_path, onnx_file), + } + } + return results + + # This should be removed + def update_override_configurations(self, config): + """Update override_configs.""" + logger.info(f"update override config with: {config}") + config = ConfigDict(**config) + self.override_configs.update(config) + + # This should moved somewhere + def _init_deploy_cfg(self) -> Union[Config, None]: + base_dir = os.path.abspath(os.path.dirname(self._task_environment.model_template.model_template_path)) + deploy_cfg_path = os.path.join(base_dir, "deployment.py") + deploy_cfg = None + if os.path.exists(deploy_cfg_path): + deploy_cfg = MPAConfig.fromfile(deploy_cfg_path) + + def patch_input_preprocessing(deploy_cfg): + normalize_cfg = get_configs_by_pairs( + self._recipe_cfg.data.test.pipeline, + dict(type="Normalize"), + ) + assert len(normalize_cfg) == 1 + normalize_cfg = normalize_cfg[0] + + options = dict(flags=[], args={}) + # NOTE: OTX loads image in RGB format + # so that `to_rgb=True` means a format change to BGR instead. + # Conventionally, OpenVINO IR expects a image in BGR format + # but OpenVINO IR under OTX assumes a image in RGB format. + # + # `to_rgb=True` -> a model was trained with images in BGR format + # and a OpenVINO IR needs to reverse input format from RGB to BGR + # `to_rgb=False` -> a model was trained with images in RGB format + # and a OpenVINO IR does not need to do a reverse + if normalize_cfg.get("to_rgb", False): + options["flags"] += ["--reverse_input_channels"] + # value must be a list not a tuple + if normalize_cfg.get("mean", None) is not None: + options["args"]["--mean_values"] = list(normalize_cfg.get("mean")) + if normalize_cfg.get("std", None) is not None: + options["args"]["--scale_values"] = list(normalize_cfg.get("std")) + + # fill default + backend_config = deploy_cfg.backend_config + if backend_config.get("mo_options") is None: + backend_config.mo_options = ConfigDict() + mo_options = backend_config.mo_options + if mo_options.get("args") is None: + mo_options.args = ConfigDict() + if mo_options.get("flags") is None: + mo_options.flags = [] + + # already defiend options have higher priority + options["args"].update(mo_options.args) + mo_options.args = ConfigDict(options["args"]) + # make sure no duplicates + mo_options.flags.extend(options["flags"]) + mo_options.flags = list(set(mo_options.flags)) + + patch_input_preprocessing(deploy_cfg) + if not deploy_cfg.backend_config.get("model_inputs", []): + raise NotImplementedError("Video recognition task must specify model input info in deployment.py") + + return deploy_cfg + + # These need to be moved somewhere + def _update_caching_modules(self, data_cfg: Config) -> None: + def _find_max_num_workers(cfg: dict): + num_workers = [0] + for key, value in cfg.items(): + if key == "workers_per_gpu" and isinstance(value, int): + num_workers += [value] + elif isinstance(value, dict): + num_workers += [_find_max_num_workers(value)] + + return max(num_workers) + + def _get_mem_cache_size(): + if not hasattr(self._hyperparams.algo_backend, "mem_cache_size"): + return 0 + + return self._hyperparams.algo_backend.mem_cache_size + + max_num_workers = _find_max_num_workers(data_cfg) + mem_cache_size = _get_mem_cache_size() + + mode = "multiprocessing" if max_num_workers > 0 else "singleprocessing" + caching.MemCacheHandlerSingleton.create(mode, mem_cache_size) + + update_or_add_custom_hook( + self._recipe_cfg, + ConfigDict(type="MemCacheHook", priority="VERY_LOW"), + ) diff --git a/otx/algorithms/action/adapters/mmaction/utils/__init__.py b/otx/algorithms/action/adapters/mmaction/utils/__init__.py index 897311310b8..788b89a8b30 100644 --- a/otx/algorithms/action/adapters/mmaction/utils/__init__.py +++ b/otx/algorithms/action/adapters/mmaction/utils/__init__.py @@ -3,8 +3,7 @@ # Copyright (C) 2022 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -from .config_utils import patch_config, prepare_for_training, set_data_classes from .det_eval_utils import det_eval from .export_utils import Exporter -__all__ = ["patch_config", "set_data_classes", "prepare_for_training", "det_eval", "Exporter"] +__all__ = ["det_eval", "Exporter"] diff --git a/otx/algorithms/action/adapters/mmaction/utils/config_utils.py b/otx/algorithms/action/adapters/mmaction/utils/config_utils.py deleted file mode 100644 index ee6ad31d73e..00000000000 --- a/otx/algorithms/action/adapters/mmaction/utils/config_utils.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Collection of utils for task implementation in Action Task.""" - -# Copyright (C) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. - -from typing import List, Union - -from mmcv.utils import Config, ConfigDict - -from otx.algorithms.common.adapters.mmcv.utils import ( - get_data_cfg, - patch_data_pipeline, - prepare_work_dir, -) -from otx.api.entities.datasets import DatasetEntity -from otx.api.entities.label import LabelEntity -from otx.api.entities.model_template import TaskType - - -def patch_config(config: Config, data_pipeline_path: str, work_dir: str, task_type: TaskType): - """Patch recipe config suitable to mmaction.""" - # FIXME omnisource is hard coded - config.omnisource = None - config.work_dir = work_dir - patch_data_pipeline(config, data_pipeline_path) - if task_type == TaskType.ACTION_CLASSIFICATION: - _patch_cls_datasets(config) - elif task_type == TaskType.ACTION_DETECTION: - _patch_det_dataset(config) - else: - raise NotImplementedError(f"{task_type} is not supported in action task") - - -def _patch_cls_datasets(config: Config): - """Patch cls dataset config suitable to mmaction.""" - - assert "data" in config - for subset in ("train", "val", "test", "unlabeled"): - cfg = config.data.get(subset, None) - if not cfg: - continue - cfg.type = "OTXActionClsDataset" - cfg.otx_dataset = None - cfg.labels = None - - -def _patch_det_dataset(config: Config): - """Patch det dataset config suitable to mmaction.""" - assert "data" in config - for subset in ("train", "val", "test", "unlabeled"): - cfg = config.data.get(subset, None) - if not cfg: - continue - cfg.type = "OTXActionDetDataset" - - -def set_data_classes(config: Config, labels: List[LabelEntity], task_type: TaskType): - """Setter data classes into config.""" - for subset in ("train", "val", "test"): - cfg = get_data_cfg(config, subset) - cfg.labels = labels - - # FIXME classification head name is hard-coded - if task_type == TaskType.ACTION_CLASSIFICATION: - config.model["cls_head"].num_classes = len(labels) - elif task_type == TaskType.ACTION_DETECTION: - config.model["roi_head"]["bbox_head"].num_classes = len(labels) + 1 - if len(labels) < 5: - config.model["roi_head"]["bbox_head"]["topk"] = len(labels) - 1 - - -def prepare_for_training( - config: Union[Config, ConfigDict], - train_dataset: DatasetEntity, - val_dataset: DatasetEntity, -) -> Config: - """Prepare configs for training phase.""" - prepare_work_dir(config) - data_train = get_data_cfg(config) - data_train.otx_dataset = train_dataset - config.data.val.otx_dataset = val_dataset - return config diff --git a/otx/algorithms/action/tasks/openvino.py b/otx/algorithms/action/adapters/openvino/task.py similarity index 99% rename from otx/algorithms/action/tasks/openvino.py rename to otx/algorithms/action/adapters/openvino/task.py index 03a1c4d2e28..4676598c79b 100644 --- a/otx/algorithms/action/tasks/openvino.py +++ b/otx/algorithms/action/adapters/openvino/task.py @@ -29,6 +29,7 @@ from compression.graph import load_model, save_model from compression.graph.model_utils import compress_model_weights, get_nodes_by_type from compression.pipeline.initializer import create_pipeline +from mmcv.utils import ProgressBar from otx.algorithms.action.adapters.openvino import ( ActionOVClsDataLoader, @@ -201,6 +202,7 @@ def infer( height = self.inferencer.model.h dataloader = get_ovdataloader(dataset, self.task_type, clip_len, width, height) dataset_size = len(dataloader) + prog_bar = ProgressBar(len(dataloader)) for i, data in enumerate(dataloader): prediction = self.inferencer.predict(data) if isinstance(dataloader, ActionOVClsDataLoader): @@ -208,6 +210,8 @@ def infer( else: dataloader.add_prediction(data, prediction) update_progress_callback(int(i / dataset_size * 100)) + prog_bar.update() + print("") return dataset def evaluate(self, output_resultset: ResultSetEntity, evaluation_metric: Optional[str] = None): diff --git a/otx/algorithms/action/configs/base/configuration.py b/otx/algorithms/action/configs/base/configuration.py index 240d26e68c7..c9183ccdd37 100644 --- a/otx/algorithms/action/configs/base/configuration.py +++ b/otx/algorithms/action/configs/base/configuration.py @@ -53,6 +53,11 @@ class __LearningParameters(BaseConfig.BaseLearningParameters): visible_in_ui=True, ) + @attrs + class __Postprocessing(BaseConfig.BasePostprocessing): + header = string_attribute("Postprocessing") + description = header + @attrs class __NNCFOptimization(BaseConfig.BaseNNCFOptimization): header = string_attribute("Optimization by NNCF") @@ -71,6 +76,7 @@ class __AlgoBackend(BaseConfig.BaseAlgoBackendParameters): description = header learning_parameters = add_parameter_group(__LearningParameters) + postprocessing = add_parameter_group(__Postprocessing) nncf_optimization = add_parameter_group(__NNCFOptimization) pot_parameters = add_parameter_group(__POTParameter) algo_backend = add_parameter_group(__AlgoBackend) diff --git a/otx/algorithms/action/configs/classification/base/supervised.py b/otx/algorithms/action/configs/classification/base/supervised.py new file mode 100644 index 00000000000..2eee1e0e3c1 --- /dev/null +++ b/otx/algorithms/action/configs/classification/base/supervised.py @@ -0,0 +1,53 @@ +"""Base supervised learning recipe for video classification.""" + +# Copyright (C) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. + +# pylint: disable=invalid-name + +seed = 2 + +evaluation = dict(interval=1, metrics=["top_k_accuracy", "mean_class_accuracy"], final_metric="mean_class_accuracy") + +optimizer = dict( + type="AdamW", + lr=0.001, + weight_decay=0.0001, +) + +optimizer_config = dict(grad_clip=dict(max_norm=40.0, norm_type=2)) +lr_config = dict( + policy="step", + step=5, + warmup="linear", + warmup_by_epoch=True, + warmup_iters=3, +) + +# runtime settings +checkpoint_config = dict(interval=1) +log_config = dict( + interval=10, + hooks=[ + dict(type="TextLoggerHook", ignore_last=False), + ], +) +# runtime settings +log_level = "INFO" +workflow = [("train", 1)] + +find_unused_parameters = False +gpu_ids = range(0, 1) + +dist_params = dict(backend="nccl") diff --git a/otx/algorithms/action/configs/classification/movinet/data_pipeline.py b/otx/algorithms/action/configs/classification/movinet/data_pipeline.py index 869caa9291b..8acd0ef5785 100644 --- a/otx/algorithms/action/configs/classification/movinet/data_pipeline.py +++ b/otx/algorithms/action/configs/classification/movinet/data_pipeline.py @@ -16,7 +16,6 @@ # pylint: disable=invalid-name # dataset settings -seed = 2 dataset_type = "RawframeDataset" img_norm_cfg = dict(mean=[0.0, 0.0, 0.0], std=[255.0, 255.0, 255.0], to_bgr=False) diff --git a/otx/algorithms/action/configs/classification/movinet/model.py b/otx/algorithms/action/configs/classification/movinet/model.py index a8b3e7cb89c..257d3cbbfdd 100644 --- a/otx/algorithms/action/configs/classification/movinet/model.py +++ b/otx/algorithms/action/configs/classification/movinet/model.py @@ -16,6 +16,8 @@ # pylint: disable=invalid-name +_base_ = ["../base/supervised.py"] + num_classes = 400 model = dict( type="MoViNetRecognizer", @@ -32,34 +34,5 @@ test_cfg=dict(average_clips="prob"), ) - -evaluation = dict(interval=1, metrics=["top_k_accuracy", "mean_class_accuracy"], final_metric="mean_class_accuracy") - -optimizer = dict( - type="AdamW", - lr=0.003, - weight_decay=0.0001, -) - -optimizer_config = dict(grad_clip=dict(max_norm=40.0, norm_type=2)) -lr_config = dict(policy="CosineAnnealing", min_lr=0) -total_epochs = 5 - -# runtime settings -checkpoint_config = dict(interval=1) -log_config = dict( - interval=10, - hooks=[ - dict(type="TextLoggerHook", ignore_last=False), - ], -) -# runtime settings -log_level = "INFO" -workflow = [("train", 1)] - -find_unused_parameters = False -gpu_ids = range(0, 1) - -dist_params = dict(backend="nccl") resume_from = None load_from = "https://github.com/Atze00/MoViNet-pytorch/blob/main/weights/modelA0_statedict_v3?raw=true" diff --git a/otx/algorithms/action/configs/classification/movinet/template.yaml b/otx/algorithms/action/configs/classification/movinet/template.yaml index c57ee80574c..b2184a1027d 100644 --- a/otx/algorithms/action/configs/classification/movinet/template.yaml +++ b/otx/algorithms/action/configs/classification/movinet/template.yaml @@ -12,8 +12,8 @@ framework: OTXAction v2.9.1 # Task implementations. entrypoints: - base: otx.algorithms.action.tasks.ActionTrainTask - openvino: otx.algorithms.action.tasks.ActionOpenVINOTask + base: otx.algorithms.action.adapters.mmaction.task.MMActionTask + openvino: otx.algorithms.action.adapters.openvino.task.ActionOpenVINOTask # Capabilities. capabilities: diff --git a/otx/algorithms/action/configs/classification/x3d/model.py b/otx/algorithms/action/configs/classification/x3d/model.py index 83b41a78124..0598ead78c7 100644 --- a/otx/algorithms/action/configs/classification/x3d/model.py +++ b/otx/algorithms/action/configs/classification/x3d/model.py @@ -16,6 +16,8 @@ # pylint: disable=invalid-name +_base_ = ["../base/supervised.py"] + num_classes = 400 num_samples = 12 model = dict( @@ -29,34 +31,6 @@ test_cfg=dict(average_clips="prob"), ) -evaluation = dict(interval=1, metrics=["top_k_accuracy", "mean_class_accuracy"], final_metric="mean_class_accuracy") - -optimizer = dict( - type="AdamW", - lr=0.001, - weight_decay=0.0001, -) - -optimizer_config = dict(grad_clip=dict(max_norm=40.0, norm_type=2)) -lr_config = dict(policy="step", step=5) -total_epochs = 5 - -# runtime settings -checkpoint_config = dict(interval=1) -log_config = dict( - interval=10, - hooks=[ - dict(type="TextLoggerHook", ignore_last=False), - ], -) -# runtime settings -log_level = "INFO" -workflow = [("train", 1)] - -find_unused_parameters = False -gpu_ids = range(0, 1) - -dist_params = dict(backend="nccl") resume_from = None load_from = ( "https://download.openmmlab.com/mmaction/recognition/x3d/facebook/" diff --git a/otx/algorithms/action/configs/classification/x3d/template.yaml b/otx/algorithms/action/configs/classification/x3d/template.yaml index f4b63ebfb1c..e2f025cec63 100644 --- a/otx/algorithms/action/configs/classification/x3d/template.yaml +++ b/otx/algorithms/action/configs/classification/x3d/template.yaml @@ -12,8 +12,8 @@ framework: OTXAction v2.9.1 # Task implementations. entrypoints: - base: otx.algorithms.action.tasks.ActionTrainTask - openvino: otx.algorithms.action.tasks.ActionOpenVINOTask + base: otx.algorithms.action.adapters.mmaction.task.MMActionTask + openvino: otx.algorithms.action.adapters.openvino.task.ActionOpenVINOTask # Capabilities. capabilities: diff --git a/otx/algorithms/action/configs/detection/base/supervised.py b/otx/algorithms/action/configs/detection/base/supervised.py new file mode 100644 index 00000000000..c4b6d5a7491 --- /dev/null +++ b/otx/algorithms/action/configs/detection/base/supervised.py @@ -0,0 +1,45 @@ +"""Supervised learning settings for video actor localization.""" + +# Copyright (C) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. + +# pylint: disable=invalid-name + +optimizer = dict(type="SGD", lr=0.1, momentum=0.9, weight_decay=1e-5) +optimizer_config = dict(grad_clip=dict(max_norm=40, norm_type=2)) + +lr_config = dict( + policy="CosineAnnealing", + by_epoch=False, + min_lr=0, + warmup="linear", + warmup_by_epoch=True, + warmup_iters=2, + warmup_ratio=0.1, +) +checkpoint_config = dict(interval=1) +workflow = [("train", 1)] +evaluation = dict(interval=1, save_best="mAP@0.5IOU", final_metric="mAP@0.5IOU") +log_config = dict( + interval=10, + hooks=[ + dict(type="TextLoggerHook"), + ], +) +dist_params = dict(backend="nccl") +log_level = "INFO" +find_unused_parameters = False +# Temporary solution, gpu_ids is not used in otx +gpu_ids = [0] +seed = 2 diff --git a/otx/algorithms/action/configs/detection/x3d_fast_rcnn/model.py b/otx/algorithms/action/configs/detection/x3d_fast_rcnn/model.py index ae2288382b3..4d2ff6aec1e 100644 --- a/otx/algorithms/action/configs/detection/x3d_fast_rcnn/model.py +++ b/otx/algorithms/action/configs/detection/x3d_fast_rcnn/model.py @@ -16,6 +16,8 @@ # pylint: disable=invalid-name +_base_ = ["../base/supervised.py"] + # model setting model = dict( type="AVAFastRCNN", @@ -38,36 +40,8 @@ test_cfg=dict(rcnn=dict(action_thr=0.002)), ) -optimizer = dict(type="SGD", lr=0.1, momentum=0.9, weight_decay=1e-5) -optimizer_config = dict(grad_clip=dict(max_norm=40, norm_type=2)) - -lr_config = dict( - policy="CosineAnnealing", - by_epoch=False, - min_lr=0, - warmup="linear", - warmup_by_epoch=True, - warmup_iters=2, - warmup_ratio=0.1, -) -checkpoint_config = dict(interval=1) -workflow = [("train", 1)] -evaluation = dict(interval=1, save_best="mAP@0.5IOU", final_metric="mAP@0.5IOU") -log_config = dict( - interval=10, - hooks=[ - dict(type="TextLoggerHook"), - ], -) -dist_params = dict(backend="nccl") -log_level = "INFO" -work_dir = "logs/x3d_kinetics_pretrained_ava_rgb/cosine/" load_from = ( "https://download.openmmlab.com/mmaction/recognition/x3d/facebook/" "x3d_m_facebook_16x5x1_kinetics400_rgb_20201027-3f42382a.pth" ) resume_from = None -find_unused_parameters = False -# Temporary solution, gpu_ids is not used in otx -gpu_ids = [0] -seed = 2 diff --git a/otx/algorithms/action/configs/detection/x3d_fast_rcnn/template.yaml b/otx/algorithms/action/configs/detection/x3d_fast_rcnn/template.yaml index b1d204d6cb0..85d9027b589 100644 --- a/otx/algorithms/action/configs/detection/x3d_fast_rcnn/template.yaml +++ b/otx/algorithms/action/configs/detection/x3d_fast_rcnn/template.yaml @@ -12,8 +12,8 @@ framework: OTXAction v2.9.1 # Task implementations. entrypoints: - base: otx.algorithms.action.tasks.ActionTrainTask - openvino: otx.algorithms.action.tasks.ActionOpenVINOTask + base: otx.algorithms.action.adapters.mmaction.task.MMActionTask + openvino: otx.algorithms.action.adapters.openvino.task.ActionOpenVINOTask # Capabilities. capabilities: diff --git a/otx/algorithms/action/task.py b/otx/algorithms/action/task.py new file mode 100644 index 00000000000..6ec491dbfc4 --- /dev/null +++ b/otx/algorithms/action/task.py @@ -0,0 +1,452 @@ +"""Task of OTX Video Recognition.""" + +# Copyright (C) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. + +import io +import os +from abc import ABC, abstractmethod +from typing import Any, Dict, Iterable, List, Optional, Union + +import numpy as np +import torch +from mmcv.utils import ConfigDict + +from otx.algorithms.action.configs.base import ActionConfig +from otx.algorithms.common.tasks.base_task import TRAIN_TYPE_DIR_PATH, OTXTask +from otx.algorithms.common.utils.callback import ( + InferenceProgressCallback, + TrainingProgressCallback, +) +from otx.algorithms.common.utils.logger import get_logger +from otx.api.configuration import cfg_helper +from otx.api.configuration.helper.utils import config_to_bytes, ids_to_strings +from otx.api.entities.annotation import Annotation +from otx.api.entities.datasets import DatasetEntity +from otx.api.entities.explain_parameters import ExplainParameters +from otx.api.entities.inference_parameters import InferenceParameters +from otx.api.entities.metrics import ( + BarChartInfo, + BarMetricsGroup, + CurveMetric, + LineChartInfo, + LineMetricsGroup, + MetricsGroup, + ScoreMetric, + VisualizationType, +) +from otx.api.entities.model import ( + ModelEntity, + ModelFormat, + ModelOptimizationType, + ModelPrecision, +) +from otx.api.entities.model_template import TaskType +from otx.api.entities.result_media import ResultMediaEntity +from otx.api.entities.resultset import ResultSetEntity +from otx.api.entities.scored_label import ScoredLabel +from otx.api.entities.shapes.rectangle import Rectangle +from otx.api.entities.subset import Subset +from otx.api.entities.task_environment import TaskEnvironment +from otx.api.entities.tensor import TensorEntity +from otx.api.entities.train_parameters import TrainParameters, default_progress_callback +from otx.api.serialization.label_mapper import label_schema_to_bytes +from otx.api.usecases.evaluation.accuracy import Accuracy +from otx.api.usecases.evaluation.f_measure import FMeasure +from otx.api.usecases.evaluation.metrics_helper import MetricsHelper +from otx.api.usecases.tasks.interfaces.export_interface import ExportType +from otx.api.utils.vis_utils import get_actmap + +logger = get_logger() + + +class OTXActionTask(OTXTask, ABC): + """Task class for OTX action.""" + + # pylint: disable=too-many-instance-attributes, too-many-locals + def __init__(self, task_environment: TaskEnvironment, output_path: Optional[str] = None): + super().__init__(task_environment, output_path) + self._task_config = ActionConfig + self._hyperparams: ConfigDict = task_environment.get_hyper_parameters(self._task_config) + self._train_type = self._hyperparams.algo_backend.train_type + self._model_dir = os.path.join( + os.path.abspath(os.path.dirname(self._task_environment.model_template.model_template_path)), + TRAIN_TYPE_DIR_PATH[self._train_type.name], + ) + + if hasattr(self._hyperparams, "postprocessing") and hasattr( + self._hyperparams.postprocessing, "confidence_threshold" + ): + self.confidence_threshold = self._hyperparams.postprocessing.confidence_threshold + else: + self.confidence_threshold = 0.0 + + if task_environment.model is not None: + self._load_model() + + self.data_pipeline_path = os.path.join(self._model_dir, "data_pipeline.py") + + def train( + self, dataset: DatasetEntity, output_model: ModelEntity, train_parameters: Optional[TrainParameters] = None + ): + """Train function for OTX action task. + + Actual training is processed by _train_model fucntion + """ + logger.info("train()") + # Check for stop signal when training has stopped. + # If should_stop is true, training was cancelled and no new + if self._should_stop: + logger.info("Training cancelled.") + self._should_stop = False + self._is_training = False + return + + # Set OTX LoggerHook & Time Monitor + if train_parameters: + update_progress_callback = train_parameters.update_progress + else: + update_progress_callback = default_progress_callback + self._time_monitor = TrainingProgressCallback(update_progress_callback) + + results = self._train_model(dataset) + + # Check for stop signal when training has stopped. If should_stop is true, training was cancelled and no new + if self._should_stop: + logger.info("Training cancelled.") + self._should_stop = False + self._is_training = False + return + + # get output model + model_ckpt = results.get("final_ckpt") + if model_ckpt is None: + logger.error("cannot find final checkpoint from the results.") + return + # update checkpoint to the newly trained model + self._model_ckpt = model_ckpt + + # get prediction on validation set + self._is_training = False + val_dataset = dataset.get_subset(Subset.VALIDATION) + val_preds, val_performance = self._infer_model(val_dataset, InferenceParameters(is_evaluation=True)) + + preds_val_dataset = val_dataset.with_empty_annotations() + if self._task_type == TaskType.ACTION_CLASSIFICATION: + self._add_cls_predictions_to_dataset(val_preds, preds_val_dataset) + elif self._task_type == TaskType.ACTION_DETECTION: + self._add_det_predictions_to_dataset(val_preds, preds_val_dataset, 0.0) + + result_set = ResultSetEntity( + model=output_model, + ground_truth_dataset=val_dataset, + prediction_dataset=preds_val_dataset, + ) + + metric: Union[Accuracy, FMeasure] + + if self._task_type == TaskType.ACTION_CLASSIFICATION: + metric = MetricsHelper.compute_accuracy(result_set) + if self._task_type == TaskType.ACTION_DETECTION: + if self._hyperparams.postprocessing.result_based_confidence_threshold: + best_confidence_threshold = None + logger.info("Adjusting the confidence threshold") + metric = MetricsHelper.compute_f_measure(result_set, vary_confidence_threshold=True) + if metric.best_confidence_threshold: + best_confidence_threshold = metric.best_confidence_threshold.value + if best_confidence_threshold is None: + raise ValueError("Cannot compute metrics: Invalid confidence threshold!") + logger.info(f"Setting confidence threshold to {best_confidence_threshold} based on results") + self.confidence_threshold = best_confidence_threshold + else: + metric = MetricsHelper.compute_f_measure(result_set, vary_confidence_threshold=False) + + # compose performance statistics + performance = metric.get_performance() + performance.dashboard_metrics.extend(self._generate_training_metrics(self._learning_curves, val_performance)) + logger.info(f"Final model performance: {performance}") + # save resulting model + self.save_model(output_model) + output_model.performance = performance + logger.info("train done.") + + @abstractmethod + def _train_model(self, dataset: DatasetEntity): + """Train model and return the results.""" + raise NotImplementedError + + def infer( + self, + dataset: DatasetEntity, + inference_parameters: Optional[InferenceParameters] = None, + ) -> DatasetEntity: + """Main infer function.""" + logger.info("infer()") + + update_progress_callback = default_progress_callback + if inference_parameters is not None: + update_progress_callback = inference_parameters.update_progress # type: ignore + + self._time_monitor = InferenceProgressCallback(len(dataset), update_progress_callback) + # If confidence threshold is adaptive then up-to-date value should be stored in the model + # and should not be changed during inference. Otherwise user-specified value should be taken. + if not self._hyperparams.postprocessing.result_based_confidence_threshold: + self.confidence_threshold = self._hyperparams.postprocessing.confidence_threshold + logger.info(f"Confidence threshold {self.confidence_threshold}") + + prediction_results, _ = self._infer_model(dataset, inference_parameters) + + if self._task_type == TaskType.ACTION_CLASSIFICATION: + self._add_cls_predictions_to_dataset(prediction_results, dataset) + elif self._task_type == TaskType.ACTION_DETECTION: + self._add_det_predictions_to_dataset(prediction_results, dataset, self.confidence_threshold) + logger.info("Inference completed") + return dataset + + @abstractmethod + def _infer_model( + self, + dataset: DatasetEntity, + inference_parameters: Optional[InferenceParameters] = None, + ): + """Get inference results from dataset.""" + raise NotImplementedError + + def export( + self, + export_type: ExportType, + output_model: ModelEntity, + precision: ModelPrecision = ModelPrecision.FP32, + dump_features: bool = True, + ): + """Export function of OTX Task.""" + if dump_features: + raise NotImplementedError( + "Feature dumping is not implemented for the action task." + "The saliency maps and representation vector outputs will not be dumped in the exported model." + ) + + # copied from OTX inference_task.py + logger.info("Exporting the model") + if export_type != ExportType.OPENVINO: + raise RuntimeError(f"not supported export type {export_type}") + output_model.model_format = ModelFormat.OPENVINO + output_model.optimization_type = ModelOptimizationType.MO + output_model.has_xai = dump_features + + results = self._export_model(precision, dump_features) + + outputs = results.get("outputs") + logger.debug(f"results of run_task = {outputs}") + if outputs is None: + raise RuntimeError(results.get("msg")) + + bin_file = outputs.get("bin") + xml_file = outputs.get("xml") + onnx_file = outputs.get("onnx") + + if xml_file is None or bin_file is None or onnx_file is None: + raise RuntimeError("invalid status of exporting. bin and xml should not be None") + with open(bin_file, "rb") as f: + output_model.set_data("openvino.bin", f.read()) + with open(xml_file, "rb") as f: + output_model.set_data("openvino.xml", f.read()) + with open(onnx_file, "rb") as f: + output_model.set_data("model.onnx", f.read()) + output_model.set_data( + "confidence_threshold", + np.array([self.confidence_threshold], dtype=np.float32).tobytes(), + ) + output_model.set_data("config.json", config_to_bytes(self._hyperparams)) + output_model.precision = self._precision + output_model.optimization_methods = self._optimization_methods + output_model.has_xai = dump_features + output_model.set_data( + "label_schema.json", + label_schema_to_bytes(self._task_environment.label_schema), + ) + logger.info("Exporting completed") + + @abstractmethod + def _export_model(self, precision: ModelPrecision, dump_features: bool = True): + raise NotImplementedError + + def explain( + self, + dataset: DatasetEntity, + explain_parameters: Optional[ExplainParameters] = None, + ) -> DatasetEntity: + """Main explain function of OTX Task.""" + raise NotImplementedError("Video recognition task don't support otx explain yet.") + + def evaluate( + self, + output_resultset: ResultSetEntity, + evaluation_metric: Optional[str] = None, + ): + """Evaluate function of OTX Action Task.""" + logger.info("called evaluate()") + if evaluation_metric is not None: + logger.warning( + f"Requested to use {evaluation_metric} metric, " "but parameter is ignored. Use F-measure instead." + ) + self._remove_empty_frames(output_resultset.ground_truth_dataset) + + metric: Union[Accuracy, FMeasure] + if self._task_type == TaskType.ACTION_CLASSIFICATION: + metric = MetricsHelper.compute_accuracy(output_resultset) + if self._task_type == TaskType.ACTION_DETECTION: + metric = MetricsHelper.compute_f_measure(output_resultset) + performance = metric.get_performance() + logger.info(f"Final model performance: {str(performance)}") + output_resultset.performance = metric.get_performance() + logger.info("Evaluation completed") + + def _remove_empty_frames(self, dataset: DatasetEntity): + """Remove empty frame for action detection dataset.""" + remove_indices = [] + for idx, item in enumerate(dataset): + if item.get_metadata()[0].data.is_empty_frame: + remove_indices.append(idx) + dataset.remove_at_indices(remove_indices) + + def _add_cls_predictions_to_dataset(self, prediction_results: Iterable, dataset: DatasetEntity): + """Loop over dataset again to assign predictions. Convert from MM format to OTX format.""" + prediction_results = list(prediction_results) + video_info: Dict[str, int] = {} + for dataset_item in dataset: + video_id = dataset_item.get_metadata()[0].data.video_id + if video_id not in video_info: + video_info[video_id] = len(video_info) + for dataset_item in dataset: + video_id = dataset_item.get_metadata()[0].data.video_id + all_results, feature_vector, saliency_map = prediction_results[video_info[video_id]] + item_labels = [] + label = ScoredLabel(label=self._labels[all_results.argmax()], probability=all_results.max()) + item_labels.append(label) + dataset_item.append_labels(item_labels) + + if feature_vector is not None: + active_score = TensorEntity(name="representation_vector", numpy=feature_vector.reshape(-1)) + dataset_item.append_metadata_item(active_score, model=self._task_environment.model) + + if saliency_map is not None: + saliency_map = get_actmap(saliency_map, (dataset_item.width, dataset_item.height)) + saliency_map_media = ResultMediaEntity( + name="Saliency Map", + type="saliency_map", + annotation_scene=dataset_item.annotation_scene, + numpy=saliency_map, + roi=dataset_item.roi, + ) + dataset_item.append_metadata_item(saliency_map_media, model=self._task_environment.model) + + def _add_det_predictions_to_dataset( + self, prediction_results: Iterable, dataset: DatasetEntity, confidence_threshold: float = 0.05 + ): + self._remove_empty_frames(dataset) + for dataset_item, (all_results, feature_vector, saliency_map) in zip(dataset, prediction_results): + shapes = [] + for label_idx, detections in enumerate(all_results): + for i in range(detections.shape[0]): + probability = float(detections[i, 4]) + coords = detections[i, :4] + + if probability < confidence_threshold: + continue + if coords[3] - coords[1] <= 0 or coords[2] - coords[0] <= 0: + continue + + assigned_label = [ScoredLabel(self._labels[label_idx], probability=probability)] + shapes.append( + Annotation( + Rectangle(x1=coords[0], y1=coords[1], x2=coords[2], y2=coords[3]), + labels=assigned_label, + ) + ) + dataset_item.append_annotations(shapes) + + if feature_vector is not None: + active_score = TensorEntity(name="representation_vector", numpy=feature_vector.reshape(-1)) + dataset_item.append_metadata_item(active_score, model=self._task_environment.model) + + if saliency_map is not None: + saliency_map = get_actmap(saliency_map, (dataset_item.width, dataset_item.height)) + saliency_map_media = ResultMediaEntity( + name="Saliency Map", + type="saliency_map", + annotation_scene=dataset_item.annotation_scene, + numpy=saliency_map, + roi=dataset_item.roi, + ) + dataset_item.append_metadata_item(saliency_map_media, model=self._task_environment.model) + + @staticmethod + # TODO Implement proper function for action classification + def _generate_training_metrics(learning_curves, scores, metric_name="mAP") -> Iterable[MetricsGroup[Any, Any]]: + """Get Training metrics (epochs & scores). + + Parses the mmaction logs to get metrics from the latest training run + :return output List[MetricsGroup] + """ + output: List[MetricsGroup] = [] + + # Learning curves. + for key, curve in learning_curves.items(): + len_x, len_y = len(curve.x), len(curve.y) + if len_x != len_y: + logger.warning(f"Learning curve {key} has inconsistent number of coordinates ({len_x} vs {len_y}.") + len_x = min(len_x, len_y) + curve.x = curve.x[:len_x] + curve.y = curve.y[:len_x] + metric_curve = CurveMetric( + xs=np.nan_to_num(curve.x).tolist(), + ys=np.nan_to_num(curve.y).tolist(), + name=key, + ) + visualization_info = LineChartInfo(name=key, x_axis_label="Epoch", y_axis_label=key) + output.append(LineMetricsGroup(metrics=[metric_curve], visualization_info=visualization_info)) + + # Final mAP value on the validation set. + output.append( + BarMetricsGroup( + metrics=[ScoreMetric(value=scores, name=f"{metric_name}")], + visualization_info=BarChartInfo("Validation score", visualization_type=VisualizationType.RADIAL_BAR), + ) + ) + + return output + + def save_model(self, output_model: ModelEntity): + """Save best model weights in ActionTrainTask.""" + logger.info("called save_model") + buffer = io.BytesIO() + hyperparams_str = ids_to_strings(cfg_helper.convert(self._hyperparams, dict, enum_to_str=True)) + labels = {label.name: label.color.rgb_tuple for label in self._labels} + model_ckpt = torch.load(self._model_ckpt) + modelinfo = { + "model": model_ckpt["state_dict"], + "config": hyperparams_str, + "labels": labels, + "confidence_threshold": self.confidence_threshold, + "VERSION": 1, + } + + torch.save(modelinfo, buffer) + output_model.set_data("weights.pth", buffer.getvalue()) + output_model.set_data( + "label_schema.json", + label_schema_to_bytes(self._task_environment.label_schema), + ) + output_model.precision = self._precision diff --git a/otx/algorithms/action/tasks/__init__.py b/otx/algorithms/action/tasks/__init__.py deleted file mode 100644 index 3cb9994da8c..00000000000 --- a/otx/algorithms/action/tasks/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Task Initialization of OTX Action Task.""" - -# Copyright (C) 2022 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. - -from .inference import ActionInferenceTask -from .openvino import ActionOpenVINOTask -from .train import ActionTrainTask - -__all__ = [ - "ActionInferenceTask", - "ActionOpenVINOTask", - "ActionTrainTask", -] diff --git a/otx/algorithms/action/tasks/inference.py b/otx/algorithms/action/tasks/inference.py deleted file mode 100644 index 8cb71a1ab90..00000000000 --- a/otx/algorithms/action/tasks/inference.py +++ /dev/null @@ -1,463 +0,0 @@ -"""Inference Task of OTX Action Task.""" - -# Copyright (C) 2022 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. - -import copy -import io -import os -import warnings -from typing import Dict, Iterable, Optional, Tuple - -import numpy as np -import torch -from mmaction.datasets import build_dataloader, build_dataset -from mmaction.models import build_model -from mmaction.utils import get_root_logger -from mmcv.parallel import MMDataParallel -from mmcv.runner import load_checkpoint, load_state_dict -from mmcv.utils import Config - -from otx.algorithms.action.adapters.mmaction import ( - Exporter, - patch_config, - set_data_classes, -) -from otx.algorithms.action.configs.base import ActionConfig -from otx.algorithms.common.adapters.mmcv.utils import prepare_for_testing -from otx.algorithms.common.tasks.training_base import BaseTask -from otx.algorithms.common.utils.callback import InferenceProgressCallback -from otx.api.entities.annotation import Annotation -from otx.api.entities.datasets import DatasetEntity -from otx.api.entities.inference_parameters import InferenceParameters -from otx.api.entities.model import ( - ModelEntity, - ModelFormat, - ModelOptimizationType, - ModelPrecision, -) -from otx.api.entities.model_template import TaskType -from otx.api.entities.result_media import ResultMediaEntity -from otx.api.entities.resultset import ResultSetEntity -from otx.api.entities.scored_label import ScoredLabel -from otx.api.entities.shapes.rectangle import Rectangle -from otx.api.entities.task_environment import TaskEnvironment -from otx.api.entities.tensor import TensorEntity -from otx.api.entities.train_parameters import default_progress_callback -from otx.api.serialization.label_mapper import label_schema_to_bytes -from otx.api.usecases.evaluation.metrics_helper import MetricsHelper -from otx.api.usecases.tasks.interfaces.evaluate_interface import IEvaluationTask -from otx.api.usecases.tasks.interfaces.export_interface import ExportType, IExportTask -from otx.api.usecases.tasks.interfaces.inference_interface import IInferenceTask -from otx.api.usecases.tasks.interfaces.unload_interface import IUnload -from otx.api.utils.vis_utils import get_actmap - -logger = get_root_logger() - - -# pylint: disable=too-many-locals, unused-argument -class ActionInferenceTask(BaseTask, IInferenceTask, IExportTask, IEvaluationTask, IUnload): - """Inference Task Implementation of OTX Action Task.""" - - def __init__(self, task_environment: TaskEnvironment, **kwargs): - super().__init__(ActionConfig, task_environment, **kwargs) - self.deploy_cfg = None - - def infer( - self, - dataset: DatasetEntity, - inference_parameters: Optional[InferenceParameters] = None, - ) -> DatasetEntity: - """Main infer function of OTX Action Task.""" - logger.info("infer()") - - if inference_parameters: - update_progress_callback = inference_parameters.update_progress - else: - update_progress_callback = default_progress_callback - - self._time_monitor = InferenceProgressCallback(len(dataset), update_progress_callback) - - def pre_hook(module, inp): - self._time_monitor.on_test_batch_begin(None, None) - - def hook(module, inp, out): - self._time_monitor.on_test_batch_end(None, None) - - if self._recipe_cfg is None: - self._init_task() - if self._model: - with self._model.register_forward_pre_hook(pre_hook), self._model.register_forward_hook(hook): - prediction_results, _ = self._infer_model(dataset, inference_parameters) - # TODO Load _add_predictions_to_dataset function from self._task_type - if self._task_type == TaskType.ACTION_CLASSIFICATION: - self._add_predictions_to_dataset(prediction_results, dataset) - elif self._task_type == TaskType.ACTION_DETECTION: - self._add_det_predictions_to_dataset(prediction_results, dataset) - logger.info("Inference completed") - else: - raise Exception("Model initialization is failed") - return dataset - - def _initialize_post_hook(self, options=None): - """Procedure after inialization.""" - - if options is None: - return - - if "deploy_cfg" in options: - self.deploy_cfg = options["deploy_cfg"] - - def _infer_model( - self, dataset: DatasetEntity, inference_parameters: Optional[InferenceParameters] = None - ) -> Tuple[Iterable, Optional[float]]: - """Inference wrapper. - - This method triggers the inference and returns `prediction_results` zipped with prediction results, - feature vectors, and saliency maps. `metric` is returned as a float value if InferenceParameters.is_evaluation - is set to true, otherwise, `None` is returned. - - Args: - dataset (DatasetEntity): the validation or test dataset to be inferred with - inference_parameters (Optional[InferenceParameters], optional): Option to run evaluation or not. - If `InferenceParameters.is_evaluation=True` then metric is returned, otherwise, both metric and - saliency maps are empty. Defaults to None. - - Returns: - Tuple[Iterable, float]: Iterable prediction results for each sample and metric for on the given dataset - """ - if self._recipe_cfg is None: - raise Exception("Recipe config is not initialized properly") - - dump_features = False - dump_saliency_map = False - - test_config = prepare_for_testing(self._recipe_cfg, dataset) - mm_test_dataset = build_dataset(test_config.data.test) - # TODO Get batch size and num_gpus autometically - batch_size = 1 - mm_test_dataloader = build_dataloader( - mm_test_dataset, - videos_per_gpu=batch_size, - workers_per_gpu=test_config.data.workers_per_gpu, - num_gpus=1, - dist=False, - shuffle=False, - ) - - self._model.eval() - if torch.cuda.is_available(): - eval_model = MMDataParallel(self._model.cuda(test_config.gpu_ids[0]), device_ids=test_config.gpu_ids) - else: - eval_model = MMDataParallel(self._model) - - eval_predictions = [] - feature_vectors = [] - saliency_maps = [] - - def dump_features_hook(): - raise NotImplementedError("get_feature_vector function for mmaction is not implemented") - - # pylint: disable=unused-argument - def dummy_dump_features_hook(model, inp, out): - feature_vectors.append(None) - - def dump_saliency_hook(): - raise NotImplementedError("get_saliency_map for mmaction is not implemented") - - # pylint: disable=unused-argument - def dummy_dump_saliency_hook(model, inp, out): - saliency_maps.append(None) - - feature_vector_hook = dump_features_hook if dump_features else dummy_dump_features_hook - saliency_map_hook = dump_saliency_hook if dump_saliency_map else dummy_dump_saliency_hook - - # Use a single gpu for testing. Set in both mm_test_dataloader and eval_model - with eval_model.module.backbone.register_forward_hook(feature_vector_hook): - with eval_model.module.backbone.register_forward_hook(saliency_map_hook): - for data in mm_test_dataloader: - with torch.no_grad(): - result = eval_model(return_loss=False, **data) - eval_predictions.extend(result) - - # hard-code way to remove EvalHook args - for key in ["interval", "tmpdir", "start", "gpu_collect", "save_best", "rule", "dynamic_intervals"]: - self._recipe_cfg.evaluation.pop(key, None) - - metric = None - metric_name = self._recipe_cfg.evaluation.final_metric - if inference_parameters: - if inference_parameters.is_evaluation: - metric = mm_test_dataset.evaluate(eval_predictions, **self._recipe_cfg.evaluation)[metric_name] - - assert len(eval_predictions) == len(feature_vectors), f"{len(eval_predictions)} != {len(feature_vectors)}" - assert len(eval_predictions) == len(saliency_maps), f"{len(eval_predictions)} != {len(saliency_maps)}" - predictions = zip(eval_predictions, feature_vectors, saliency_maps) - - return predictions, metric - - # pylint: disable=attribute-defined-outside-init - def _init_task(self, **kwargs): - # FIXME: Temporary remedy for CVS-88098 - self._initialize(kwargs) - logger.info(f"running task... kwargs = {kwargs}") - if self._recipe_cfg is None: - raise RuntimeError("'config' is not initialized yet. call prepare() method before calling this method") - - self._model = self._load_model(self._task_environment.model) - - def _load_model(self, model: ModelEntity): - if self._recipe_cfg is None: - raise Exception("Recipe config is not initialized properly") - if model is not None: - # If a model has been trained and saved for the task already, create empty model and load weights here - buffer = io.BytesIO(model.get_data("weights.pth")) - model_data = torch.load(buffer, map_location=torch.device("cpu")) - - self.confidence_threshold: float = model_data.get("confidence_threshold", self.confidence_threshold) - model = self._create_model(self._recipe_cfg, from_scratch=True) - - try: - load_state_dict(model, model_data["model"]) - - # It prevent model from being overwritten - if "load_from" in self._recipe_cfg: - self._recipe_cfg.load_from = None - - logger.info("Loaded model weights from Task Environment") - logger.info(f"Model architecture: {self._model_name}") - for name, weights in model.named_parameters(): - if not torch.isfinite(weights).all(): - logger.info(f"Invalid weights in: {name}. Recreate model from pre-trained weights") - model = self._create_model(self._recipe_cfg, from_scratch=False) - return model - - except BaseException as ex: - raise ValueError("Could not load the saved model. The model file structure is invalid.") from ex - else: - # If there is no trained model yet, create model with pretrained weights as defined in the model config - # file. - model = self._create_model(self._recipe_cfg, from_scratch=False) - logger.info( - f"No trained model in project yet. Created new model with '{self._model_name}' " - f"architecture and general-purpose pretrained weights." - ) - return model - - @staticmethod - def _create_model(config: Config, from_scratch: bool = False): - """Creates a model, based on the configuration in config. - - :param config: mmaction configuration from which the model has to be built - :param from_scratch: bool, if True does not load any weights - - :return model: ModelEntity in training mode - """ - model_cfg = copy.deepcopy(config.model) - - init_from = None if from_scratch else config.get("load_from", None) - logger.warning(init_from) - if init_from is not None: - # No need to initialize backbone separately, if all weights are provided. - # model_cfg.pretrained = None - logger.warning("build model") - model = build_model(model_cfg) - # Load all weights. - logger.warning("load checkpoint") - load_checkpoint(model, init_from, map_location="cpu") - else: - logger.warning("build model") - model = build_model(model_cfg) - return model - - def evaluate(self, output_resultset: ResultSetEntity, evaluation_metric: Optional[str] = None): - """Evaluate function of OTX Action Task.""" - logger.info("called evaluate()") - self._remove_empty_frames(output_resultset.ground_truth_dataset) - metric = self._get_metric(output_resultset) - performance = metric.get_performance() - logger.info(f"Final model performance: {str(performance)}") - output_resultset.performance = metric.get_performance() - logger.info("Evaluation completed") - - def _get_metric(self, output_resultset: ResultSetEntity): - if self._task_type == TaskType.ACTION_CLASSIFICATION: - return MetricsHelper.compute_accuracy(output_resultset) - if self._task_type == TaskType.ACTION_DETECTION: - return MetricsHelper.compute_f_measure(output_resultset) - raise NotImplementedError(f"{self._task_type} is not supported in action task") - - def _remove_empty_frames(self, dataset: DatasetEntity): - """Remove empty frame for action detection dataset.""" - remove_indices = [] - for idx, item in enumerate(dataset): - if item.get_metadata()[0].data.is_empty_frame: - remove_indices.append(idx) - dataset.remove_at_indices(remove_indices) - - def unload(self): - """Unload the task.""" - if self._work_dir_is_temp: - self._delete_scratch_space() - - def export( - self, - export_type: ExportType, - output_model: ModelEntity, - precision: ModelPrecision = ModelPrecision.FP32, - dump_features: bool = False, - ): - """Export function of OTX Action Task.""" - if dump_features: - raise NotImplementedError( - "Feature dumping is not implemented for the anomaly task." - "The saliency maps and representation vector outputs will not be dumped in the exported model." - ) - - # copied from OTX inference_task.py - logger.info("Exporting the model") - if export_type != ExportType.OPENVINO: - raise RuntimeError(f"not supported export type {export_type}") - output_model.model_format = ModelFormat.OPENVINO - output_model.optimization_type = ModelOptimizationType.MO - output_model.has_xai = dump_features - self._init_task(export=True, dump_features=dump_features) - - self._precision[0] = precision - half_precision = precision == ModelPrecision.FP16 - - try: - from torch.jit._trace import TracerWarning - - warnings.filterwarnings("ignore", category=TracerWarning) - exporter = Exporter( - self._recipe_cfg, - self._model.state_dict(), - self.deploy_cfg, - f"{self._output_path}/openvino", - half_precision, - ) - exporter.export() - bin_file = [f for f in os.listdir(self._output_path) if f.endswith(".bin")][0] - xml_file = [f for f in os.listdir(self._output_path) if f.endswith(".xml")][0] - onnx_file = [f for f in os.listdir(self._output_path) if f.endswith(".onnx")][0] - with open(os.path.join(self._output_path, bin_file), "rb") as f: - output_model.set_data("openvino.bin", f.read()) - with open(os.path.join(self._output_path, xml_file), "rb") as f: - output_model.set_data("openvino.xml", f.read()) - with open(os.path.join(self._output_path, onnx_file), "rb") as file: - output_model.set_data("model.onnx", file.read()) - output_model.set_data( - "confidence_threshold", np.array([self.confidence_threshold], dtype=np.float32).tobytes() - ) - output_model.precision = self._precision - output_model.optimization_methods = self._optimization_methods - except Exception as ex: - raise RuntimeError("Optimization was unsuccessful.") from ex - output_model.set_data("label_schema.json", label_schema_to_bytes(self._task_environment.label_schema)) - logger.info("Exporting completed") - - def _init_recipe_hparam(self) -> dict: - configs = super()._init_recipe_hparam() - configs.data.videos_per_gpu = configs.data.pop("samples_per_gpu", None) # type: ignore[attr-defined] - self._recipe_cfg.total_epochs = configs.runner.max_epochs # type: ignore[attr-defined] - # FIXME lr_config variables are hard-coded - if hasattr(configs, "lr_config") and hasattr(configs["lr_config"], "warmup_iters"): - self._recipe_cfg.lr_config.warmup = "linear" # type: ignore[attr-defined] - self._recipe_cfg.lr_config.warmup_by_epoch = True # type: ignore[attr-defined] - configs["use_adaptive_interval"] = self._hyperparams.learning_parameters.use_adaptive_interval - return configs - - def _init_recipe(self): - logger.info("called _init_recipe()") - recipe_root = os.path.abspath(os.path.dirname(self.template_file_path)) - recipe = os.path.join(recipe_root, "model.py") - self._recipe_cfg = Config.fromfile(recipe) - patch_config(self._recipe_cfg, self.data_pipeline_path, self._output_path, self._task_type) - set_data_classes(self._recipe_cfg, self._labels, self._task_type) - logger.info(f"initialized recipe = {recipe}") - - def _init_model_cfg(self): - model_cfg = Config.fromfile(os.path.join(self._model_dir, "model.py")) - return model_cfg - - def _add_predictions_to_dataset(self, prediction_results: Iterable, dataset: DatasetEntity): - """Loop over dataset again to assign predictions. Convert from MM format to OTX format.""" - prediction_results = list(prediction_results) - video_info: Dict[str, int] = {} - for dataset_item in dataset: - video_id = dataset_item.get_metadata()[0].data.video_id - if video_id not in video_info: - video_info[video_id] = len(video_info) - for dataset_item in dataset: - video_id = dataset_item.get_metadata()[0].data.video_id - all_results, feature_vector, saliency_map = prediction_results[video_info[video_id]] - item_labels = [] - label = ScoredLabel(label=self._labels[all_results.argmax()], probability=all_results.max()) - item_labels.append(label) - dataset_item.append_labels(item_labels) - - if feature_vector is not None: - active_score = TensorEntity(name="representation_vector", numpy=feature_vector.reshape(-1)) - dataset_item.append_metadata_item(active_score, model=self._task_environment.model) - - if saliency_map is not None: - saliency_map = get_actmap(saliency_map, (dataset_item.width, dataset_item.height)) - saliency_map_media = ResultMediaEntity( - name="Saliency Map", - type="saliency_map", - annotation_scene=dataset_item.annotation_scene, - numpy=saliency_map, - roi=dataset_item.roi, - ) - dataset_item.append_metadata_item(saliency_map_media, model=self._task_environment.model) - - def _add_det_predictions_to_dataset(self, prediction_results: Iterable, dataset: DatasetEntity): - confidence_threshold = 0.05 - self._remove_empty_frames(dataset) - for dataset_item, (all_results, feature_vector, saliency_map) in zip(dataset, prediction_results): - shapes = [] - for label_idx, detections in enumerate(all_results): - for i in range(detections.shape[0]): - probability = float(detections[i, 4]) - coords = detections[i, :4] - - if probability < confidence_threshold: - continue - if coords[3] - coords[1] <= 0 or coords[2] - coords[0] <= 0: - continue - - assigned_label = [ScoredLabel(self._labels[label_idx], probability=probability)] - shapes.append( - Annotation( - Rectangle(x1=coords[0], y1=coords[1], x2=coords[2], y2=coords[3]), - labels=assigned_label, - ) - ) - dataset_item.append_annotations(shapes) - - if feature_vector is not None: - active_score = TensorEntity(name="representation_vector", numpy=feature_vector.reshape(-1)) - dataset_item.append_metadata_item(active_score, model=self._task_environment.model) - - if saliency_map is not None: - saliency_map = get_actmap(saliency_map, (dataset_item.width, dataset_item.height)) - saliency_map_media = ResultMediaEntity( - name="Saliency Map", - type="saliency_map", - annotation_scene=dataset_item.annotation_scene, - numpy=saliency_map, - roi=dataset_item.roi, - ) - dataset_item.append_metadata_item(saliency_map_media, model=self._task_environment.model) diff --git a/otx/algorithms/action/tasks/train.py b/otx/algorithms/action/tasks/train.py deleted file mode 100644 index a10d6f398ae..00000000000 --- a/otx/algorithms/action/tasks/train.py +++ /dev/null @@ -1,237 +0,0 @@ -"""Train Task of OTX Action Task.""" - -# Copyright (C) 2022 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. - -import io -import os -from glob import glob -from typing import Any, Iterable, List, Optional - -import numpy as np -import torch -from mmaction.apis import train_model -from mmaction.datasets import build_dataset -from mmaction.utils import get_root_logger - -from otx.algorithms.action.adapters.mmaction.utils import prepare_for_training -from otx.algorithms.common.utils import TrainingProgressCallback -from otx.api.configuration import cfg_helper -from otx.api.configuration.helper.utils import ids_to_strings -from otx.api.entities.datasets import DatasetEntity -from otx.api.entities.inference_parameters import InferenceParameters -from otx.api.entities.metrics import ( - BarChartInfo, - BarMetricsGroup, - CurveMetric, - LineChartInfo, - LineMetricsGroup, - MetricsGroup, - ScoreMetric, - VisualizationType, -) -from otx.api.entities.model import ModelEntity -from otx.api.entities.model_template import TaskType -from otx.api.entities.resultset import ResultSetEntity -from otx.api.entities.subset import Subset -from otx.api.entities.train_parameters import TrainParameters, default_progress_callback -from otx.api.serialization.label_mapper import label_schema_to_bytes -from otx.api.usecases.tasks.interfaces.training_interface import ITrainingTask - -from .inference import ActionInferenceTask - -logger = get_root_logger() - - -# pylint: disable=too-many-locals, too-many-instance-attributes, too-many-ancestors -class ActionTrainTask(ActionInferenceTask, ITrainingTask): - """Train Task Implementation of OTX Action Task.""" - - def save_model(self, output_model: ModelEntity): - """Save best model weights in ActionTrainTask.""" - logger.info("called save_model") - buffer = io.BytesIO() - hyperparams_str = ids_to_strings(cfg_helper.convert(self._hyperparams, dict, enum_to_str=True)) - labels = {label.name: label.color.rgb_tuple for label in self._labels} - model_ckpt = torch.load(self._model_ckpt) - modelinfo = { - "model": model_ckpt["state_dict"], - "config": hyperparams_str, - "labels": labels, - "confidence_threshold": self.confidence_threshold, - "VERSION": 1, - } - - torch.save(modelinfo, buffer) - output_model.set_data("weights.pth", buffer.getvalue()) - output_model.set_data( - "label_schema.json", - label_schema_to_bytes(self._task_environment.label_schema), - ) - output_model.precision = self._precision - - def cancel_training(self): - """Cancel training function in ActionTrainTask. - - Sends a cancel training signal to gracefully stop the optimizer. The signal consists of creating a - '.stop_training' file in the current work_dir. The runner checks for this file periodically. - The stopping mechanism allows stopping after each iteration, but validation will still be carried out. Stopping - will therefore take some time. - """ - logger.info("Cancel training requested.") - self._should_stop = True - if self.cancel_interface is not None: - self.cancel_interface.cancel() - else: - logger.info("but training was not started yet. reserved it to cancel") - self.reserved_cancel = True - - def train( - self, - dataset: DatasetEntity, - output_model: ModelEntity, - train_parameters: Optional[TrainParameters] = None, - ): - """Train function in ActionTrainTask.""" - logger.info("train()") - # Check for stop signal when training has stopped. - # If should_stop is true, training was cancelled and no new - if self._should_stop: - logger.info("Training cancelled.") - self._should_stop = False - self._is_training = False - return - - # Set OTX LoggerHook & Time Monitor - if train_parameters: - update_progress_callback = train_parameters.update_progress - else: - update_progress_callback = default_progress_callback - self._time_monitor = TrainingProgressCallback(update_progress_callback) - - self._is_training = True - self._init_task() - - if self._recipe_cfg is None: - raise Exception("Recipe config is not initialized properly") - - results = self._train_model(dataset) - - # Check for stop signal when training has stopped. If should_stop is true, training was cancelled and no new - if self._should_stop: - logger.info("Training cancelled.") - self._should_stop = False - self._is_training = False - return - - self._get_output_model(results) - performance = self._get_final_eval_results(dataset, output_model) - - # save resulting model - self.save_model(output_model) - output_model.performance = performance - self._is_training = False - logger.info("train done.") - - def _train_model(self, dataset: DatasetEntity): - if self._recipe_cfg is None: - raise Exception("Recipe config does not initialize properly!") - train_dataset = dataset.get_subset(Subset.TRAINING) - val_dataset = dataset.get_subset(Subset.VALIDATION) - training_config = prepare_for_training(self._recipe_cfg, train_dataset, val_dataset) - mm_train_dataset = build_dataset(training_config.data.train) - logger.info("Start training") - self._model.train() - # FIXME runner is built inside of train_model funciton, it is hard to change runner's type - train_model(model=self._model, dataset=mm_train_dataset, cfg=training_config, validate=True) - checkpoint_file_path = glob(os.path.join(self._recipe_cfg.work_dir, "best*pth"))[0] - if len(checkpoint_file_path) == 0: - checkpoint_file_path = os.path.join(self._recipe_cfg.work_dir, "latest.pth") - logger.info(f"Use {checkpoint_file_path} for final model weights") - - return {"final_ckpt": checkpoint_file_path} - - def _get_output_model(self, results): - model_ckpt = results.get("final_ckpt") - if model_ckpt is None: - logger.error("cannot find final checkpoint from the results.") - return - # update checkpoint to the newly trained model - self._model_ckpt = model_ckpt - self._model.load_state_dict(torch.load(self._model_ckpt)["state_dict"]) - - def _get_final_eval_results(self, dataset, output_model): - logger.info("Final Evaluation") - val_dataset = dataset.get_subset(Subset.VALIDATION) - val_preds, val_map = self._infer_model(val_dataset, InferenceParameters(is_evaluation=True)) - - preds_val_dataset = val_dataset.with_empty_annotations() - # TODO Load _add_predictions_to_dataset function from self._task_type - if self._task_type == TaskType.ACTION_CLASSIFICATION: - self._add_predictions_to_dataset(val_preds, preds_val_dataset) - elif self._task_type == TaskType.ACTION_DETECTION: - self._add_det_predictions_to_dataset(val_preds, preds_val_dataset) - - result_set = ResultSetEntity( - model=output_model, - ground_truth_dataset=val_dataset, - prediction_dataset=preds_val_dataset, - ) - - metric = self._get_metric(result_set) - - # compose performance statistics - performance = metric.get_performance() - metric_name = self._recipe_cfg.evaluation.final_metric - performance.dashboard_metrics.extend( - ActionTrainTask._generate_training_metrics(self._learning_curves, val_map, metric_name) - ) - logger.info(f"Final model performance: {str(performance)}") - return performance - - @staticmethod - # TODO Implement proper function for action classification - def _generate_training_metrics(learning_curves, scores, metric_name) -> Iterable[MetricsGroup[Any, Any]]: - """Get Training metrics (epochs & scores). - - Parses the mmaction logs to get metrics from the latest training run - :return output List[MetricsGroup] - """ - output: List[MetricsGroup] = [] - - # Learning curves. - for key, curve in learning_curves.items(): - len_x, len_y = len(curve.x), len(curve.y) - if len_x != len_y: - logger.warning(f"Learning curve {key} has inconsistent number of coordinates ({len_x} vs {len_y}.") - len_x = min(len_x, len_y) - curve.x = curve.x[:len_x] - curve.y = curve.y[:len_x] - metric_curve = CurveMetric( - xs=np.nan_to_num(curve.x).tolist(), - ys=np.nan_to_num(curve.y).tolist(), - name=key, - ) - visualization_info = LineChartInfo(name=key, x_axis_label="Epoch", y_axis_label=key) - output.append(LineMetricsGroup(metrics=[metric_curve], visualization_info=visualization_info)) - - # Final mAP value on the validation set. - output.append( - BarMetricsGroup( - metrics=[ScoreMetric(value=scores, name=f"{metric_name}")], - visualization_info=BarChartInfo("Validation score", visualization_type=VisualizationType.RADIAL_BAR), - ) - ) - - return output diff --git a/otx/algorithms/common/adapters/mmcv/hooks/__init__.py b/otx/algorithms/common/adapters/mmcv/hooks/__init__.py index f4d8c8c8c2d..97c98c670d3 100644 --- a/otx/algorithms/common/adapters/mmcv/hooks/__init__.py +++ b/otx/algorithms/common/adapters/mmcv/hooks/__init__.py @@ -50,7 +50,6 @@ from .task_adapt_hook import TaskAdaptHook from .two_crop_transform_hook import TwoCropTransformHook from .unbiased_teacher_hook import UnbiasedTeacherHook -from .workflow_hook import WorkflowHook __all__ = [ "AdaptiveTrainSchedulingHook", @@ -87,5 +86,4 @@ "TaskAdaptHook", "TwoCropTransformHook", "UnbiasedTeacherHook", - "WorkflowHook", ] diff --git a/otx/algorithms/common/adapters/mmcv/hooks/workflow_hook.py b/otx/algorithms/common/adapters/mmcv/hooks/workflow_hook.py deleted file mode 100644 index 741e7e3ff16..00000000000 --- a/otx/algorithms/common/adapters/mmcv/hooks/workflow_hook.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Workflow hooks.""" -# Copyright (C) 2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -import copy -import datetime -import json - -from mmcv.utils import Registry - -from otx.algorithms.common.utils.logger import get_logger - -logger = get_logger() -WORKFLOW_HOOKS = Registry("workflow_hooks") - -# pylint: disable=unused-argument - - -def build_workflow_hook(config, *args, **kwargs): - """Build a workflow hook.""" - logger.info(f"called build_workflow_hook({config})") - whook_type = config.pop("type") - # event = config.pop('event') - if whook_type not in WORKFLOW_HOOKS: - raise KeyError(f"not supported workflow hook type {whook_type}") - whook_cls = WORKFLOW_HOOKS.get(whook_type) - return whook_cls(*args, **kwargs, **config) - - -class WorkflowHook: - """Workflow hook.""" - - def __init__(self, name): - self.name = name - - def before_workflow(self, workflow, idx=-1, results=None): - """Before workflow.""" - return - - def after_workflow(self, workflow, idx=-1, results=None): - """After workflow.""" - return - - def before_stage(self, workflow, idx, results=None): - """Before stage.""" - return - - def after_stage(self, workflow, idx, results=None): - """After stage.""" - return - - -@WORKFLOW_HOOKS.register_module() -class SampleLoggingHook(WorkflowHook): - """Sample logging hook.""" - - def __init__(self, name=__name__, log_level="DEBUG"): - super().__init__(name) - self.logging = getattr(logger, log_level.lower()) - - def before_stage(self, workflow, idx, results=None): - """Before stage.""" - self.logging(f"called {self.name}.run()") - self.logging(f"stage index {idx}, results keys = {results.keys()}") - result_key = f"{self.name}|{idx}" - results[result_key] = dict(message=f"this is a sample result of the {__name__} hook") - - -@WORKFLOW_HOOKS.register_module() -class WFProfileHook(WorkflowHook): - """Workflow profile hook.""" - - def __init__(self, name=__name__, output_path=None): - super().__init__(name) - self.output_path = output_path - self.profile = dict(start=0, end=0, elapsed=0, stages=dict()) - logger.info(f"initialized {__name__}....") - - def before_workflow(self, workflow, idx=-1, results=None): - """Before workflow.""" - self.profile["start"] = datetime.datetime.now() - - def after_workflow(self, workflow, idx=-1, results=None): - """After workflow.""" - self.profile["end"] = datetime.datetime.now() - self.profile["elapsed"] = self.profile["end"] - self.profile["start"] - - str_dumps = json.dumps(self.profile, indent=2, default=str) - logger.info("** workflow profile results **") - logger.info(str_dumps) - if self.output_path is not None: - with open(self.output_path, "w") as f: # pylint: disable=unspecified-encoding - f.write(str_dumps) - - def before_stage(self, workflow, idx=-1, results=None): - """Before stage.""" - stages = self.profile.get("stages") - stages[f"{idx}"] = {} - stages[f"{idx}"]["start"] = datetime.datetime.now() - - def after_stage(self, workflow, idx=-1, results=None): - """After stage.""" - stages = self.profile.get("stages") - stages[f"{idx}"]["end"] = datetime.datetime.now() - stages[f"{idx}"]["elapsed"] = stages[f"{idx}"]["end"] - stages[f"{idx}"]["start"] - - -@WORKFLOW_HOOKS.register_module() -class AfterStageWFHook(WorkflowHook): - """After stage workflow hook.""" - - def __init__(self, name, stage_cfg_updated_callback): - self.callback = stage_cfg_updated_callback - super().__init__(name) - - def after_stage(self, workflow, idx, results=None): - """After stage.""" - logger.info(f"{__name__}: called after_stage()") - name = copy.deepcopy(workflow.stages[idx].name) - cfg = copy.deepcopy(workflow.stages[idx].cfg) - self.callback(name, cfg) diff --git a/otx/algorithms/common/adapters/mmcv/tasks/__init__.py b/otx/algorithms/common/adapters/mmcv/tasks/__init__.py index fd4acb753af..9f3efd00e8b 100644 --- a/otx/algorithms/common/adapters/mmcv/tasks/__init__.py +++ b/otx/algorithms/common/adapters/mmcv/tasks/__init__.py @@ -6,10 +6,7 @@ # flake8: noqa import os -from .builder import build, build_workflow_hook -from .stage import Stage, get_available_types from .version import __version__, get_version -from .workflow import Workflow class MPAConstants: @@ -27,10 +24,5 @@ class MPAConstants: __all__ = [ "get_version", "__version__", - "build", - "build_workflow_hook", - "Stage", - "get_available_types", - "Workflow", "MPAConstants", ] diff --git a/otx/algorithms/common/adapters/mmcv/tasks/builder.py b/otx/algorithms/common/adapters/mmcv/tasks/builder.py deleted file mode 100644 index d4736aa3c78..00000000000 --- a/otx/algorithms/common/adapters/mmcv/tasks/builder.py +++ /dev/null @@ -1,118 +0,0 @@ -"""Build workflow.""" -# Copyright (C) 2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -import copy -import os -import time - -from mmcv import Config, ConfigDict, build_from_cfg - -from otx.algorithms.common.adapters.mmcv.hooks.workflow_hook import ( - WorkflowHook, - build_workflow_hook, -) -from otx.algorithms.common.adapters.mmcv.tasks.registry import STAGES -from otx.algorithms.common.adapters.mmcv.tasks.stage import get_available_types -from otx.algorithms.common.adapters.mmcv.tasks.workflow import Workflow -from otx.algorithms.common.adapters.mmcv.utils.config_utils import MPAConfig -from otx.algorithms.common.utils.logger import config_logger, get_logger - -# from collections import defaultdict - - -logger = get_logger() - - -def __build_stage(config, common_cfg=None, index=0): - logger.info("build_stage()") - logger.debug(f"[args] config = {config}, common_cfg = {common_cfg}, index={index}") - config.type = config.type if "type" in config.keys() else "Stage" # TODO: tmp workaround code for competability - config.common_cfg = common_cfg - config.index = index - return build_from_cfg(config, STAGES) - - -def __build_workflow(config): - logger.info("build_workflow()") - logger.debug(f"[args] config = {config}") - - whooks = [] - whooks_cfg = config.get("workflow_hooks", []) - for whook_cfg in whooks_cfg: - if isinstance(whook_cfg, WorkflowHook): - whooks.append(whook_cfg) - else: - whook = build_workflow_hook(whook_cfg.copy()) - whooks.append(whook) - - output_path = config.get("output_path", "logs") - folder_name = f"{time.strftime('%Y%m%d_%H%M%S', time.localtime())}" - config.output_path = os.path.join(output_path, folder_name) - os.makedirs(config.output_path, exist_ok=True) - - # create symbolic link to the output path - symlink_dst = os.path.join(output_path, "latest") - if os.path.exists(symlink_dst): - os.unlink(symlink_dst) - os.symlink(folder_name, symlink_dst, True) - - log_level = config.get("log_level", "INFO") - config_logger(os.path.join(config.output_path, "app.log"), level=log_level) - - if not hasattr(config, "gpu_ids"): - gpu_ids = os.environ.get("CUDA_VISIBLE_DEVICES", None) - logger.info(f"CUDA_VISIBLE_DEVICES = {gpu_ids}") - if gpu_ids is not None: - if isinstance(gpu_ids, str): - config.gpu_ids = range(len(gpu_ids.split(","))) - else: - raise ValueError(f"not supported type for gpu_ids: {type(gpu_ids)}") - else: - config.gpu_ids = range(1) - - common_cfg = copy.deepcopy(config) - common_cfg.pop("stages") - if len(whooks_cfg) > 0: - common_cfg.pop("workflow_hooks") - - stages = [__build_stage(stage_cfg.copy(), common_cfg, index=i) for i, stage_cfg in enumerate(config.stages)] - return Workflow(stages, whooks) - - -def build(config, mode=None, stage_type=None, common_cfg=None): - """Build workflow.""" - logger.info("called build_recipe()") - logger.debug(f"[args] config = {config}") - - if not isinstance(config, Config): - if isinstance(config, str): - if os.path.exists(config): - config = MPAConfig.fromfile(config) - else: - logger.error(f"cannot find configuration file {config}") - raise ValueError(f"cannot find configuration file {config}") - - if hasattr(config, "stages"): - # build as workflow - return __build_workflow(config) - # build as stage - if not hasattr(config, "type"): - logger.info("seems to be passed stage yaml...") - supported_stage_types = get_available_types() - if stage_type in supported_stage_types: - cfg_dict = ConfigDict( - dict( - type=stage_type, - name=f"{stage_type}-{mode}", - mode=mode, - config=config, - index=0, - ) - ) - else: - msg = f"type {stage_type} is not in {supported_stage_types}" - logger.error(msg) - raise RuntimeError(msg) - return __build_stage(cfg_dict, common_cfg=common_cfg) diff --git a/otx/algorithms/common/adapters/mmcv/tasks/exporter_mixin.py b/otx/algorithms/common/adapters/mmcv/tasks/exporter_mixin.py deleted file mode 100644 index e03ca23c450..00000000000 --- a/otx/algorithms/common/adapters/mmcv/tasks/exporter_mixin.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Base Exporter for OTX tasks.""" -# Copyright (C) 2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -import os -import traceback - -from otx.algorithms.common.utils.logger import get_logger - -logger = get_logger() - - -class ExporterMixin: - """Exporter Mixin class for OTX export.""" - - def run(self, model_cfg, model_ckpt, data_cfg, **kwargs): # noqa: C901 - """Run export procedure.""" - self._init_logger() - logger.info("exporting the model") - mode = kwargs.get("mode", "train") - if mode not in self.mode: - logger.warning(f"Supported modes are {self.mode} but '{mode}' is given.") - return {} - - cfg = self.configure(model_cfg, model_ckpt, data_cfg, training=False, **kwargs) - logger.info("export!") - - # from torch.jit._trace import TracerWarning - # import warnings - # warnings.filterwarnings("ignore", category=TracerWarning) - precision = kwargs.pop("precision", "FP32") - if precision not in ("FP32", "FP16", "INT8"): - raise NotImplementedError - logger.info(f"Model will be exported with precision {precision}") - model_name = cfg.get("model_name", "model") - - # TODO: handle complicated pipeline - # If test dataset is a wrapper dataset - # pipeline may not include load transformation which is assumed to be included afterwards - # Here, we assume simple wrapper datasets where pipeline of the wrapper is just a consecutive one. - if cfg.data.test.get("dataset", None) or cfg.data.test.get("datasets", None): - dataset = cfg.data.test.get("dataset", cfg.data.test.get("datasets", [None])[0]) - assert dataset is not None - pipeline = dataset.get("pipeline", []) - pipeline += cfg.data.test.get("pipeline", []) - cfg.data.test.pipeline = pipeline - - model_builder = kwargs.get("model_builder") - try: - deploy_cfg = kwargs.get("deploy_cfg", None) - if deploy_cfg is not None: - self.mmdeploy_export( - cfg.work_dir, - model_builder, - precision, - cfg, - deploy_cfg, - model_name, - ) - else: - self.naive_export(cfg.work_dir, model_builder, precision, cfg, model_name) - except RuntimeError as ex: - # output_model.model_status = ModelStatus.FAILED - # raise RuntimeError('Optimization was unsuccessful.') from ex - return { - "outputs": None, - "msg": f"exception {type(ex)}: {ex}\n\n{traceback.format_exc()}", - } - - return { - "outputs": { - "bin": os.path.join(cfg.work_dir, f"{model_name}.bin"), - "xml": os.path.join(cfg.work_dir, f"{model_name}.xml"), - "onnx": os.path.join(cfg.work_dir, f"{model_name}.onnx"), - "partitioned": [ - { - "bin": os.path.join(cfg.work_dir, name.replace(".xml", ".bin")), - "xml": os.path.join(cfg.work_dir, name), - } - for name in os.listdir(cfg.work_dir) - if name.endswith(".xml") - and name != f"{model_name}.xml" - and name.replace(".xml", ".bin") in os.listdir(cfg.work_dir) - ], - }, - "msg": "", - } - - @staticmethod - def mmdeploy_export( - output_dir, - model_builder, - precision, - cfg, - deploy_cfg, - model_name="model", - ): - """Export procedure using mmdeploy backend.""" - from otx.algorithms.common.adapters.mmdeploy.apis import MMdeployExporter - - if precision == "FP16": - deploy_cfg.backend_config.mo_options.flags.append("--compress_to_fp16") - MMdeployExporter.export2openvino(output_dir, model_builder, cfg, deploy_cfg, model_name=model_name) - - @staticmethod - def naive_export(output_dir, model_builder, precision, cfg, model_name="model"): - """Export using pytorch backend.""" - raise NotImplementedError() diff --git a/otx/algorithms/common/adapters/mmcv/tasks/registry.py b/otx/algorithms/common/adapters/mmcv/tasks/registry.py index 9381f330f96..9182c760d6c 100644 --- a/otx/algorithms/common/adapters/mmcv/tasks/registry.py +++ b/otx/algorithms/common/adapters/mmcv/tasks/registry.py @@ -1,9 +1,8 @@ -"""Registry of Stages and Explainers.""" +"""Registry of Explainers.""" # Copyright (C) 2022 Intel Corporation # SPDX-License-Identifier: Apache-2.0 # from mmcv.utils import Registry -STAGES = Registry("stages") EXPLAINERS = Registry("explainers") diff --git a/otx/algorithms/common/adapters/mmcv/tasks/stage.py b/otx/algorithms/common/adapters/mmcv/tasks/stage.py deleted file mode 100644 index 8fedaabe6d0..00000000000 --- a/otx/algorithms/common/adapters/mmcv/tasks/stage.py +++ /dev/null @@ -1,550 +0,0 @@ -"""Base stage for OTX tasks.""" -# Copyright (C) 2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -import importlib -import json -import os -import os.path as osp -import random -import time -from typing import Any, Callable, Dict, List, Optional - -import mmcv -import numpy as np -import torch -from mmcv import Config, ConfigDict -from mmcv.runner import CheckpointLoader, wrap_fp16_model -from torch import distributed as dist - -from otx.algorithms.common.adapters.mmcv.utils import ( - build_dataloader, - build_dataset, - get_data_cfg, -) -from otx.algorithms.common.adapters.mmcv.utils.config_utils import ( - MPAConfig, - update_or_add_custom_hook, -) -from otx.algorithms.common.utils.logger import config_logger, get_logger - -from .registry import STAGES - -logger = get_logger() - - -def _set_random_seed(seed, deterministic=False): - """Set random seed. - - Args: - seed (int): Seed to be used. - deterministic (bool): Whether to set the deterministic option for - CUDNN backend, i.e., set `torch.backends.cudnn.deterministic` - to True and `torch.backends.cudnn.benchmark` to False. - Default: False. - """ - random.seed(seed) - np.random.seed(seed) - torch.manual_seed(seed) - torch.cuda.manual_seed_all(seed) - os.environ["PYTHONHASHSEED"] = str(seed) - logger.info(f"Training seed was set to {seed} w/ deterministic={deterministic}.") - if deterministic: - torch.backends.cudnn.deterministic = True - torch.backends.cudnn.benchmark = False - - -def get_available_types(): - """Return available type of stage.""" - types = [] - for key in STAGES.module_dict: - types.append(key) - return types - - -MODEL_TASK = {"classification": "mmcls", "detection": "mmdet", "segmentation": "mmseg"} - - -# @STAGES.register_module() -# pylint: disable=too-many-instance-attributes -class Stage: - """Class for base stage of OTX tasks.""" - - MODEL_BUILDER: Optional[Callable] = None - - # pylint: disable=too-many-branches, too-many-statements - def __init__(self, name, mode, config, common_cfg=None, index=0, **kwargs): - logger.debug(f"init stage with: {name}, {mode}, {config}, {common_cfg}, {index}, {kwargs}") - # the name of 'config' cannot be changed to such as 'config_file' - # because it is defined as 'config' in recipe file..... - self.name = name - self.mode = mode - self.index = index - self.input = kwargs.pop("input", {}) # input_map?? input_dict? just input? - self.output_keys = kwargs.pop("output", []) - self._distributed = False - self.task_adapt_type = None - self.task_adapt_op = "REPLACE" - self.org_model_classes: List[str] = [] - self.model_classes: List[str] = [] - self.data_classes: List[str] = [] - - if common_cfg is None: - common_cfg = dict(output_path="logs") - - if not isinstance(common_cfg, dict): - raise TypeError(f"common_cfg should be the type of dict but {type(common_cfg)}") - if common_cfg.get("output_path") is None: - logger.info("output_path is not set in common_cfg. set it to 'logs' as default") - common_cfg["output_path"] = "logs" - - self.output_prefix = common_cfg["output_path"] - self.output_suffix = f"stage{self.index:02d}_{self.name}" - - # # Work directory - # work_dir = os.path.join(self.output_prefix, self.output_suffix) - # mmcv.mkdir_or_exist(os.path.abspath(work_dir)) - - if isinstance(config, Config): - cfg = config - elif isinstance(config, dict): - cfg = Config(cfg_dict=config) - elif isinstance(config, str): - if os.path.exists(config): - cfg = MPAConfig.fromfile(config) - else: - err_msg = f"cannot find configuration file {config}" - logger.error(err_msg) - raise ValueError(err_msg) - else: - err_msg = "'config' argument could be one of the \ - [dictionary, Config object, or string of the cfg file path]" - logger.error(err_msg) - raise ValueError(err_msg) - - cfg.merge_from_dict(common_cfg) - - if len(kwargs) > 0: - addtional_dict = {} - logger.info("found override configurations for the stage") - for key, value in kwargs.items(): - addtional_dict[key] = value - logger.info(f"\t{key}: {value}") - cfg.merge_from_dict(addtional_dict) - - max_epochs = -1 - if hasattr(cfg, "total_epochs"): - max_epochs = cfg.pop("total_epochs") - if hasattr(cfg, "runner"): - if hasattr(cfg.runner, "max_epochs"): - if max_epochs != -1: - max_epochs = min(max_epochs, cfg.runner.max_epochs) - else: - max_epochs = cfg.runner.max_epochs - if max_epochs > 0: - if cfg.runner.max_epochs != max_epochs: - cfg.runner.max_epochs = max_epochs - logger.info(f"The maximum number of epochs is adjusted to {max_epochs}.") - if hasattr(cfg, "checkpoint_config"): - if hasattr(cfg.checkpoint_config, "interval"): - if cfg.checkpoint_config.interval > max_epochs: - logger.warning( - f"adjusted checkpoint interval from {cfg.checkpoint_config.interval} to {max_epochs} \ - since max_epoch is shorter than ckpt interval configuration" - ) - cfg.checkpoint_config.interval = max_epochs - - if cfg.get("seed", None) is not None: - _set_random_seed(cfg.seed, deterministic=cfg.get("deterministic", False)) - else: - cfg.seed = None - - # Work directory - work_dir = cfg.get("work_dir", "") - work_dir = os.path.join(self.output_prefix, work_dir if work_dir else "", self.output_suffix) - cfg.work_dir = os.path.abspath(work_dir) - logger.info(f"work dir = {cfg.work_dir}") - mmcv.mkdir_or_exist(os.path.abspath(work_dir)) - - # config logger replace hook - hook_cfg = ConfigDict(type="LoggerReplaceHook") - update_or_add_custom_hook(cfg, hook_cfg) - - self.cfg = cfg - - self.__init_device() - - def __init_device(self): - if torch.distributed.is_initialized(): - self._distributed = True - self.cfg.gpu_ids = [int(os.environ["LOCAL_RANK"])] - elif "gpu_ids" not in self.cfg: - self.cfg.gpu_ids = range(1) - elif len(self.cfg.gpu_ids) != 1: - raise ValueError("length of cfg.gpu_ids should be 1 when training with single GPU.") - - # consider "cuda" and "cpu" device only - if not torch.cuda.is_available(): - self.cfg.device = "cpu" - self.cfg.gpu_ids = range(-1, 0) - else: - self.cfg.device = "cuda" - - @property - def distributed(self): - """Return whether this is distributed running.""" - return self._distributed - - def _init_logger(self): - timestamp = time.strftime("%Y%m%d_%H%M%S", time.localtime()) - config_logger(os.path.join(self.cfg.work_dir, f"{timestamp}.log"), level=self.cfg.log_level) - logger.info(f"configured logger at {self.cfg.work_dir} with named {timestamp}.log") - return logger - - def configure_data(self, cfg, training): - """Update data configuration using image options.""" - logger.info("configure_data()") - logger.debug(f"[args] {cfg.data}") - pipeline_options = cfg.data.pop("pipeline_options", None) - if pipeline_options is not None and isinstance(pipeline_options, dict): - self._configure_split(cfg, pipeline_options, "train") - self._configure_split(cfg, pipeline_options, "val") - if not training: - self._configure_split(cfg, pipeline_options, "test") - self._configure_split(cfg, pipeline_options, "unlabeled") - - @staticmethod - def _configure_split(cfg, pipeline_options, target): - def update_transform(opt, pipeline, idx, transform): - if isinstance(opt, dict): - if "_delete_" in opt.keys() and opt.get("_delete_", False): - # if option include _delete_=True, remove this transform from pipeline - logger.info(f"configure_data: {transform['type']} is deleted") - del pipeline[idx] - return - logger.info(f"configure_data: {transform['type']} is updated with {opt}") - transform.update(**opt) - - # pylint: disable=too-many-nested-blocks - def update_config(src, pipeline_options): - logger.info(f"update_config() {pipeline_options}") - if src.get("pipeline") is not None or ( - src.get("dataset") is not None and src.get("dataset").get("pipeline") is not None - ): - if src.get("pipeline") is not None: - pipeline = src.get("pipeline", None) - else: - pipeline = src.get("dataset").get("pipeline") - if isinstance(pipeline, list): - for idx, transform in enumerate(pipeline): - for opt_key, opt in pipeline_options.items(): - if transform["type"] == opt_key: - update_transform(opt, pipeline, idx, transform) - elif isinstance(pipeline, dict): - for _, pipe in pipeline.items(): - for idx, transform in enumerate(pipe): - for opt_key, opt in pipeline_options.items(): - if transform["type"] == opt_key: - update_transform(opt, pipe, idx, transform) - else: - raise NotImplementedError(f"pipeline type of {type(pipeline)} is not supported") - else: - logger.info("no pipeline in the data split") - - split = cfg.data.get(target) - if split is not None: - if isinstance(split, list): - for sub_item in split: - update_config(sub_item, pipeline_options) - elif isinstance(split, dict): - update_config(split, pipeline_options) - else: - logger.warning(f"type of split '{target}'' should be list or dict but {type(split)}") - - def configure_ckpt(self, cfg, model_ckpt, pretrained=None): - """Patch checkpoint path for pretrained weight. - - Replace cfg.load_from to model_ckpt - Replace cfg.load_from to pretrained - Replace cfg.resume_from to cfg.load_from - """ - if model_ckpt: - cfg.load_from = self.get_model_ckpt(model_ckpt) - if pretrained and isinstance(pretrained, str): - logger.info(f"Overriding cfg.load_from -> {pretrained}") - cfg.load_from = pretrained # Overriding by stage input - if cfg.get("resume", False): - cfg.resume_from = cfg.load_from - - @staticmethod - def configure_hook(cfg): - """Update cfg.custom_hooks based on cfg.custom_hook_options.""" - - def update_hook(opt, custom_hooks, idx, hook): - """Delete of update a custom hook.""" - if isinstance(opt, dict): - if opt.get("_delete_", False): - # if option include _delete_=True, remove this hook from custom_hooks - logger.info(f"configure_hook: {hook['type']} is deleted") - del custom_hooks[idx] - else: - logger.info(f"configure_hook: {hook['type']} is updated with {opt}") - hook.update(**opt) - - custom_hook_options = cfg.pop("custom_hook_options", {}) - # logger.info(f"configure_hook() {cfg.get('custom_hooks', [])} <- {custom_hook_options}") - custom_hooks = cfg.get("custom_hooks", []) - for idx, hook in enumerate(custom_hooks): - for opt_key, opt in custom_hook_options.items(): - if hook["type"] == opt_key: - update_hook(opt, custom_hooks, idx, hook) - - @staticmethod - def configure_samples_per_gpu( - cfg: Config, - subset: str, - distributed: bool = False, - ): - """Patch samples_per_gpu settings.""" - - dataloader_cfg = cfg.data.get(f"{subset}_dataloader", ConfigDict()) - samples_per_gpu = dataloader_cfg.get("samples_per_gpu", cfg.data.get("samples_per_gpu", 1)) - - data_cfg = get_data_cfg(cfg, subset) - dataset_len = len(data_cfg.otx_dataset) - - if distributed: - dataset_len = dataset_len // dist.get_world_size() - - # set batch size as a total dataset - # if it is smaller than total dataset - if dataset_len < samples_per_gpu: - dataloader_cfg.samples_per_gpu = dataset_len - - # drop the last batch if the last batch size is 1 - # batch size of 1 is a runtime error for training batch normalization layer - if subset in ("train", "unlabeled") and dataset_len % samples_per_gpu == 1: - dataloader_cfg.drop_last = True - - cfg.data[f"{subset}_dataloader"] = dataloader_cfg - - @staticmethod - def configure_compat_cfg( - cfg: Config, - ): - """Modify config to keep the compatibility.""" - - def _configure_dataloader(cfg): - """Consume all the global dataloader config and convert them to specific dataloader config.""" - global_dataloader_cfg = {} - global_dataloader_cfg.update( - { - k: cfg.data.pop(k) - for k in list(cfg.data.keys()) - if k - not in [ - "train", - "val", - "test", - "unlabeled", - "train_dataloader", - "val_dataloader", - "test_dataloader", - "unlabeled_dataloader", - ] - } - ) - - for subset in ["train", "val", "test", "unlabeled"]: - if subset not in cfg.data: - continue - dataloader_cfg = cfg.data.get(f"{subset}_dataloader", None) - if dataloader_cfg is None: - raise AttributeError(f"{subset}_dataloader is not found in config.") - dataloader_cfg = Config(cfg_dict={**global_dataloader_cfg, **dataloader_cfg}) - cfg.data[f"{subset}_dataloader"] = dataloader_cfg - - _configure_dataloader(cfg) - - @staticmethod - def configure_fp16_optimizer(cfg: Config, distributed: bool = False): - """Configure Fp16OptimizerHook and Fp16SAMOptimizerHook.""" - - fp16_config = cfg.pop("fp16", None) - if fp16_config is not None: - optim_type = cfg.optimizer_config.get("type", "OptimizerHook") - opts: Dict[str, Any] = dict( - distributed=distributed, - **fp16_config, - ) - if optim_type == "SAMOptimizerHook": - opts["type"] = "Fp16SAMOptimizerHook" - elif optim_type == "OptimizerHook": - opts["type"] = "Fp16OptimizerHook" - else: - # does not support optimizerhook type - # let mm library handle it - cfg.fp16 = fp16_config - opts = dict() - cfg.optimizer_config.update(opts) - - @staticmethod - def configure_unlabeled_dataloader(cfg: Config, distributed: bool = False): - """Patch for loading unlabeled dataloader.""" - if "unlabeled" in cfg.data: - task_lib_module = importlib.import_module(f"{MODEL_TASK[cfg.model_task]}.datasets") - dataset_builder = getattr(task_lib_module, "build_dataset") - dataloader_builder = getattr(task_lib_module, "build_dataloader") - - dataset = build_dataset(cfg, "unlabeled", dataset_builder, consume=True) - unlabeled_dataloader = build_dataloader( - dataset, - cfg, - "unlabeled", - dataloader_builder, - distributed=distributed, - consume=True, - ) - - custom_hooks = cfg.get("custom_hooks", []) - updated = False - for custom_hook in custom_hooks: - if custom_hook["type"] == "ComposedDataLoadersHook": - custom_hook["data_loaders"] = [*custom_hook["data_loaders"], unlabeled_dataloader] - updated = True - if not updated: - custom_hooks.append( - ConfigDict( - type="ComposedDataLoadersHook", - data_loaders=unlabeled_dataloader, - ) - ) - cfg.custom_hooks = custom_hooks - - @staticmethod - def get_model_meta(cfg): - """Return model_meta.""" - ckpt_path = cfg.get("load_from", None) - meta = {} - if ckpt_path: - ckpt = CheckpointLoader.load_checkpoint(ckpt_path, map_location="cpu") - meta = ckpt.get("meta", {}) - return meta - - @staticmethod - def get_data_cfg(cfg, subset): - """Return data_cfg from cfg's subset.""" - assert subset in ["train", "val", "test"], f"Unknown subset:{subset}" - if "dataset" in cfg.data[subset]: # Concat|RepeatDataset - dataset = cfg.data[subset].dataset - while hasattr(dataset, "dataset"): - dataset = dataset.dataset - return dataset - return cfg.data[subset] - - @staticmethod - def get_data_classes(cfg): - """Return data_classes from cfg.""" - data_classes = [] - train_cfg = Stage.get_data_cfg(cfg, "train") - if "data_classes" in train_cfg: - data_classes = list(train_cfg.pop("data_classes", [])) - elif "classes" in train_cfg: - data_classes = list(train_cfg.classes) - return data_classes - - @staticmethod - def get_model_classes(cfg): - """Extract trained classes info from checkpoint file. - - MMCV-based models would save class info in ckpt['meta']['CLASSES'] - For other cases, try to get the info from cfg.model.classes (with pop()) - - Which means that model classes should be specified in model-cfg for - non-MMCV models (e.g. OMZ models) - """ - classes = [] - meta = Stage.get_model_meta(cfg) - # for MPA classification legacy compatibility - classes = meta.get("CLASSES", []) - classes = meta.get("classes", classes) - if classes is None: - classes = [] - - if len(classes) == 0: - ckpt_path = cfg.get("load_from", None) - if ckpt_path: - classes = Stage.read_label_schema(ckpt_path) - if len(classes) == 0: - classes = cfg.model.pop("classes", cfg.pop("model_classes", [])) - return classes - - @staticmethod - def get_model_ckpt(ckpt_path, new_path=None): - """Return model ckpt from ckpt_path.""" - ckpt = CheckpointLoader.load_checkpoint(ckpt_path, map_location="cpu") - if "model" in ckpt: - ckpt = ckpt["model"] - if not new_path: - new_path = ckpt_path[:-3] + "converted.pth" - torch.save(ckpt, new_path) - return new_path - return ckpt_path - - @staticmethod - def read_label_schema(ckpt_path, name_only=True, file_name="label_schema.json"): - """Read label_schema and return all classes.""" - serialized_label_schema = [] - if any(ckpt_path.endswith(extension) for extension in (".xml", ".bin", ".pth")): - label_schema_path = osp.join(osp.dirname(ckpt_path), file_name) - if osp.exists(label_schema_path): - with open(label_schema_path, encoding="UTF-8") as read_file: - serialized_label_schema = json.load(read_file) - if serialized_label_schema: - if name_only: - all_classes = [labels["name"] for labels in serialized_label_schema["all_labels"].values()] - else: - all_classes = serialized_label_schema - else: - all_classes = [] - return all_classes - - # pylint: disable=unused-argument - @staticmethod - def set_inference_progress_callback(model, cfg): - """Inferenceprogresscallback (Time Monitor enable into Infer task).""" - time_monitor = None - if cfg.get("custom_hooks", None): - time_monitor = [hook.time_monitor for hook in cfg.custom_hooks if hook.type == "OTXProgressHook"] - time_monitor = time_monitor[0] if time_monitor else None - if time_monitor is not None: - - def pre_hook(*args, **kwargs): - time_monitor.on_test_batch_begin(None, None) - - def hook(*args, **kwargs): - time_monitor.on_test_batch_end(None, None) - - model.register_forward_pre_hook(pre_hook) - model.register_forward_hook(hook) - - @classmethod - def build_model( - cls, - cfg: Config, - model_builder: Optional[Callable] = None, - *, - fp16: bool = False, - **kwargs, - ) -> torch.nn.Module: - """Build model from model_builder.""" - if model_builder is None: - model_builder = cls.MODEL_BUILDER - assert model_builder is not None - model = model_builder(cfg, **kwargs) - if bool(fp16): - wrap_fp16_model(model) - return model - - def _get_feature_module(self, model): - return model diff --git a/otx/algorithms/common/adapters/mmcv/tasks/workflow.py b/otx/algorithms/common/adapters/mmcv/tasks/workflow.py deleted file mode 100644 index b25767056b9..00000000000 --- a/otx/algorithms/common/adapters/mmcv/tasks/workflow.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Base workflow for OTX task.""" -# Copyright (C) 2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -# from datetime import datetime as dt - -from otx.algorithms.common.adapters.mmcv.utils.config_utils import copy_config - -from .stage import Stage - - -class Workflow: - """Base workflow class for OTX task.""" - - def __init__(self, stages, workflow_hooks=None): - if not isinstance(stages, list): - raise ValueError("stages parameter should be the list of Stage instance") - if len(stages) == 0: - raise ValueError("required one or more stage for the workflow") - for stage in stages: - if not isinstance(stage, Stage): - raise ValueError("stages parameter should be the list of Stage instance") - if workflow_hooks is not None and not isinstance(workflow_hooks, list): - raise ValueError("workflow_hooks should be a list") - - self.stages = stages - self.workflow_hooks = workflow_hooks - self.results = {} - self.context = {stage.name: {} for stage in stages} - - def _call_wf_hooks(self, fname, stage_idx=-1): - if self.workflow_hooks is not None: - for hook in self.workflow_hooks: - getattr(hook, fname)(self, stage_idx, self.results) - - # pylint: disable=too-many-locals - def run(self, **kwargs): - """Run workflow.""" - model_cfg = kwargs.get("model_cfg", None) - data_cfg = kwargs.get("data_cfg", None) - model_ckpt = kwargs.get("model_ckpt", None) - # output_path = kwargs.get('output_path', '.') - mode = kwargs.get("mode", "train") - ir_model_path = kwargs.get("ir_model_path", None) - ir_weight_path = kwargs.get("ir_weight_path", None) - ir_weight_init = kwargs.get("ir_weight_init", False) - - self._call_wf_hooks("before_workflow") - for i, stage in enumerate(self.stages): - self._call_wf_hooks("before_stage", i) - - # create keyword arguments that will be passed to stage.run() refer to input map defined in config - stage_kwargs = dict() - if hasattr(stage, "input"): - for arg_name, arg in stage.input.items(): - stage_name = arg.get("stage_name", None) - output_key = arg.get("output_key", None) - if stage_name is None or output_key is None: - raise ValueError(f"'stage_name' and 'output_key' attributes are required for the '{arg_name}'") - stage_kwargs[arg_name] = self.context[stage_name].get(output_key, None) - - # context will keep the results(path to model, etc) of each stage - # stage.run() returns a dict and each output data will be stored in each output key defined in config - self.context[stage.name] = stage.run( - stage_idx=i, - mode=mode, - # model_cfg and data_cfg can be changed by each stage. need to pass cloned one for the other stages - # note that mmcv's Config object manage its attributes inside of _cfg_dict so need to copy it as well - model_cfg=copy_config(model_cfg) if model_cfg is not None else model_cfg, - data_cfg=copy_config(data_cfg) if data_cfg is not None else data_cfg, - model_ckpt=model_ckpt, - # output_path=output_path+'/stage{:02d}_{}'.format(i, stage.name), - ir_model_path=ir_model_path, - ir_weight_path=ir_weight_path, - ir_weight_init=ir_weight_init, - **stage_kwargs, - ) - # TODO: save context as pickle after each stage?? - self._call_wf_hooks("after_stage", i) - self._call_wf_hooks("after_workflow") diff --git a/otx/algorithms/common/tasks/__init__.py b/otx/algorithms/common/tasks/__init__.py index a59498a6566..e0dc4c6af19 100644 --- a/otx/algorithms/common/tasks/__init__.py +++ b/otx/algorithms/common/tasks/__init__.py @@ -13,7 +13,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions # and limitations under the License. - -from .training_base import BaseTask - -__all__ = ["BaseTask"] diff --git a/otx/algorithms/segmentation/adapters/mmseg/utils/exporter.py b/otx/algorithms/segmentation/adapters/mmseg/utils/exporter.py index f5afb1d6f4e..c5ac0b90bf8 100644 --- a/otx/algorithms/segmentation/adapters/mmseg/utils/exporter.py +++ b/otx/algorithms/segmentation/adapters/mmseg/utils/exporter.py @@ -7,7 +7,6 @@ from mmcv.runner import wrap_fp16_model from otx.algorithms.common.adapters.mmcv.tasks.exporter import Exporter -from otx.algorithms.common.adapters.mmcv.tasks.registry import STAGES from otx.algorithms.common.adapters.mmdeploy.utils import sync_batchnorm_2_batchnorm from otx.algorithms.common.utils.logger import get_logger from otx.algorithms.segmentation.adapters.mmseg.utils.builder import build_segmentor @@ -15,7 +14,6 @@ logger = get_logger() -@STAGES.register_module() class SegmentationExporter(Exporter): """Exporter for OTX Segmentation using mmsegmentation training backend.""" diff --git a/tests/unit/algorithms/action/adapters/mmaction/test_task.py b/tests/unit/algorithms/action/adapters/mmaction/test_task.py new file mode 100644 index 00000000000..16d27a058a7 --- /dev/null +++ b/tests/unit/algorithms/action/adapters/mmaction/test_task.py @@ -0,0 +1,350 @@ +"""Unit Test for otx.algorithms.action.adapters.mmaction.task.""" + +# Copyright (C) 2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import os +from copy import deepcopy +from typing import Any, Dict + +import numpy as np +import pytest +import torch +from mmaction.models.backbones.x3d import X3D +from mmaction.models.recognizers.recognizer3d import Recognizer3D +from mmcv.runner import CheckpointLoader +from mmcv.utils import Config +from torch import nn + +from otx.algorithms.action.configs.base.configuration import ActionConfig +from otx.algorithms.action.adapters.mmaction.task import MMActionTask +from otx.algorithms.common.adapters.mmcv.utils.config_utils import MPAConfig +from otx.api.configuration import ConfigurableParameters +from otx.api.configuration.helper import create +from otx.api.entities.dataset_item import DatasetItemEntity +from otx.api.entities.datasets import DatasetEntity +from otx.api.entities.inference_parameters import InferenceParameters +from otx.api.entities.label import Domain +from otx.api.entities.label_schema import LabelGroup, LabelGroupType, LabelSchemaEntity +from otx.api.entities.model import ( + ModelConfiguration, + ModelEntity, + ModelFormat, + ModelOptimizationType, + ModelPrecision, +) +from otx.api.entities.model_template import InstantiationType, parse_model_template, TaskFamily, TaskType +from otx.api.entities.resultset import ResultSetEntity +from otx.api.usecases.tasks.interfaces.export_interface import ExportType +from tests.test_suite.e2e_test_system import e2e_pytest_unit +from tests.unit.algorithms.action.test_helpers import ( + init_environment, + MockModelTemplate, + generate_action_cls_otx_dataset, + generate_action_det_otx_dataset, + generate_labels, + return_inputs, +) + +DEFAULT_ACTION_CLS_DIR = os.path.join("otx/algorithms/action/configs/classification", "x3d") +DEFAULT_ACTION_DET_DIR = os.path.join("otx/algorithms/action/configs/detection", "x3d_fast_rcnn") + + +class MockModule(nn.Module): + """Mock class for nn.Module.""" + + def forward(self, inputs: Any): + return inputs + + +class MockModel(nn.Module): + """Mock class for pytorch model.""" + + def __init__(self, task_type): + super().__init__() + self.module = MockModule() + self.module.backbone = MockModule() + self.backbone = MockModule() + self.task_type = task_type + + def forward(self, return_loss: bool, imgs: DatasetItemEntity): + forward_hooks = list(self.module.backbone._forward_hooks.values()) + for hook in forward_hooks: + hook(1, 2, 3) + if self.task_type == "cls": + return np.array([[0, 0, 1]]) + return [[np.array([[0, 0, 1, 1, 0.1]]), np.array([[0, 0, 1, 1, 0.2]]), np.array([[0, 0, 1, 1, 0.7]])]] + + @staticmethod + def named_parameters(): + return {"name": torch.Tensor([0.5])}.items() + + +class MockDataset(DatasetEntity): + """Mock class for mm_dataset.""" + + def __init__(self, dataset: DatasetEntity, task_type: str): + self.dataset = dataset + self.task_type = task_type + self.CLASSES = ["1", "2", "3"] + + def evaluate(self, prediction, *args, **kwargs): + if self.task_type == "cls": + return {"mean_class_accuracy": 1.0} + else: + return {"mAP@0.5IOU": 1.0} + + +class MockDataLoader: + """Mock class for data loader.""" + + def __init__(self, dataset: DatasetEntity): + self.dataset = dataset + self.iter = iter(self.dataset) + + def __len__(self) -> int: + return len(self.dataset) + + def __next__(self) -> Dict[str, DatasetItemEntity]: + return {"imgs": next(self.iter)} + + def __iter__(self): + return self + + +class MockExporter: + """Mock class for Exporter.""" + + def __init__(self, task): + self.work_dir = task._output_path + + def export(self): + dummy_data = np.ndarray((1, 1, 1)) + with open(os.path.join(self.work_dir, "openvino.bin"), "wb") as f: + f.write(dummy_data) + with open(os.path.join(self.work_dir, "openvino.xml"), "wb") as f: + f.write(dummy_data) + with open(os.path.join(self.work_dir, "model.onnx"), "wb") as f: + f.write(dummy_data) + + +class TestMMActionTask: + """Test class for MMActionTask. + + Details are explained in each test function. + """ + + @pytest.fixture(autouse=True) + def setup(self) -> None: + self.video_len = 3 + self.frame_len = 3 + + cls_labels = generate_labels(3, Domain.ACTION_CLASSIFICATION) + self.cls_label_schema = LabelSchemaEntity() + cls_label_group = LabelGroup( + name="labels", + labels=cls_labels, + group_type=LabelGroupType.EXCLUSIVE, + ) + self.cls_label_schema.add_group(cls_label_group) + self.cls_dataset = generate_action_cls_otx_dataset(self.video_len, self.frame_len, cls_labels) + + cls_model_template = parse_model_template(os.path.join(DEFAULT_ACTION_CLS_DIR, "template.yaml")) + cls_hyper_parameters = create(cls_model_template.hyper_parameters.data) + cls_task_env = init_environment(cls_hyper_parameters, cls_model_template, self.cls_label_schema) + self.cls_task = MMActionTask(task_environment=cls_task_env) + + det_labels = generate_labels(3, Domain.ACTION_DETECTION) + self.det_label_schema = LabelSchemaEntity() + det_label_group = LabelGroup( + name="labels", + labels=det_labels, + group_type=LabelGroupType.EXCLUSIVE, + ) + self.det_label_schema.add_group(det_label_group) + self.det_dataset = generate_action_det_otx_dataset(self.video_len, self.frame_len, det_labels)[0] + + det_model_template = parse_model_template(os.path.join(DEFAULT_ACTION_DET_DIR, "template.yaml")) + det_hyper_parameters = create(det_model_template.hyper_parameters.data) + det_task_env = init_environment(det_hyper_parameters, det_model_template, self.det_label_schema) + self.det_task = MMActionTask(task_environment=det_task_env) + + @e2e_pytest_unit + def test_build_model(self, mocker) -> None: + """Test build_model function.""" + _weight = torch.randn([24, 3, 1, 3, 3]) + mocker.patch.object( + CheckpointLoader, + "load_checkpoint", + return_value={"model": {"state_dict": {"backbone.conv1_s.conv.weight": _weight}}}, + ) + _mock_recipe_cfg = MPAConfig.fromfile(os.path.join(DEFAULT_ACTION_CLS_DIR, "model.py")) + model = self.cls_task.build_model(_mock_recipe_cfg, True) + assert isinstance(model, Recognizer3D) + assert isinstance(model.backbone, X3D) + assert torch.all(model.state_dict()["backbone.conv1_s.conv.weight"] == _weight) + + @e2e_pytest_unit + def test_train(self, mocker) -> None: + """Test train function.""" + mocker.patch( + "otx.algorithms.action.adapters.mmaction.task.build_dataset", + return_value=MockDataset(self.cls_dataset, "cls"), + ) + mocker.patch( + "otx.algorithms.action.adapters.mmaction.task.build_dataloader", + return_value=MockDataLoader(self.cls_dataset), + ) + mocker.patch.object(MMActionTask, "build_model", return_value=MockModel("cls")) + mocker.patch( + "otx.algorithms.action.adapters.mmaction.task.patch_data_pipeline", + return_value=True, + ) + mocker.patch( + "otx.algorithms.action.adapters.mmaction.task.build_data_parallel", + return_value=MockModel("cls"), + ) + mocker.patch( + "otx.algorithms.action.adapters.mmaction.task.train_model", + return_value=True, + ) + mocker.patch("torch.load", return_value={"state_dict": np.ndarray([1, 1, 1])}) + + _config = ModelConfiguration(ActionConfig(), self.cls_label_schema) + output_model = ModelEntity(self.cls_dataset, _config) + self.cls_task.train(self.cls_dataset, output_model) + output_model.performance == 1.0 + + mocker.patch( + "otx.algorithms.action.adapters.mmaction.task.build_dataset", + return_value=MockDataset(self.det_dataset, "det"), + ) + mocker.patch( + "otx.algorithms.action.adapters.mmaction.task.build_dataloader", + return_value=MockDataLoader(self.det_dataset), + ) + mocker.patch.object(MMActionTask, "build_model", return_value=MockModel("det")) + mocker.patch( + "otx.algorithms.action.adapters.mmaction.task.patch_data_pipeline", + return_value=True, + ) + mocker.patch( + "otx.algorithms.action.adapters.mmaction.task.build_data_parallel", + return_value=MockModel("det"), + ) + _config = ModelConfiguration(ActionConfig(), self.det_label_schema) + output_model = ModelEntity(self.det_dataset, _config) + self.det_task.train(self.det_dataset, output_model) + output_model.performance == 1.0 + + @e2e_pytest_unit + def test_infer(self, mocker) -> None: + """Test infer function. + + + 1. Create mock model for action classification + 2. Create mock recipe for action classification + 3. Run infer funciton + 4. Check whether inference results are added to output + 5. Do 1 - 4 for action detection + """ + mocker.patch( + "otx.algorithms.action.adapters.mmaction.task.build_dataset", + return_value=MockDataset(self.cls_dataset, "cls"), + ) + mocker.patch( + "otx.algorithms.action.adapters.mmaction.task.build_dataloader", + return_value=MockDataLoader(self.cls_dataset), + ) + mocker.patch.object(MMActionTask, "build_model", return_value=MockModel("cls")) + mocker.patch( + "otx.algorithms.action.adapters.mmaction.task.patch_data_pipeline", + return_value=True, + ) + mocker.patch( + "otx.algorithms.action.adapters.mmaction.task.build_data_parallel", + return_value=MockModel("cls"), + ) + + inference_parameters = InferenceParameters(is_evaluation=True) + outputs = self.cls_task.infer(self.cls_dataset, inference_parameters) + for output in outputs: + assert len(output.get_annotations()[0].get_labels()) == 2 + + mocker.patch( + "otx.algorithms.action.adapters.mmaction.task.build_dataset", + return_value=MockDataset(self.det_dataset, "det"), + ) + mocker.patch( + "otx.algorithms.action.adapters.mmaction.task.build_dataloader", + return_value=MockDataLoader(self.det_dataset), + ) + mocker.patch.object(MMActionTask, "build_model", return_value=MockModel("det")) + mocker.patch( + "otx.algorithms.action.adapters.mmaction.task.patch_data_pipeline", + return_value=True, + ) + mocker.patch( + "otx.algorithms.action.adapters.mmaction.task.build_data_parallel", + return_value=MockModel("det"), + ) + inference_parameters = InferenceParameters(is_evaluation=True) + outputs = self.det_task.infer(self.det_dataset, inference_parameters) + for output in outputs: + assert len(output.get_annotations()) == 2 + + @e2e_pytest_unit + def test_evaluate(self) -> None: + """Test evaluate function.""" + _config = ModelConfiguration(ActionConfig(), self.cls_label_schema) + _model = ModelEntity(self.cls_dataset, _config) + resultset = ResultSetEntity(_model, self.cls_dataset, self.cls_dataset) + self.cls_task.evaluate(resultset) + assert resultset.performance.score.value == 1.0 + + @e2e_pytest_unit + def test_evaluate_with_empty_annot(self) -> None: + """Test evaluate function with empty_annot.""" + _config = ModelConfiguration(ActionConfig(), self.cls_label_schema) + _model = ModelEntity(self.cls_dataset, _config) + resultset = ResultSetEntity(_model, self.cls_dataset, self.cls_dataset.with_empty_annotations()) + self.cls_task.evaluate(resultset) + assert resultset.performance.score.value == 0.0 + + @e2e_pytest_unit + def test_evaluate_det(self) -> None: + """Test evaluate function for action detection.""" + _config = ModelConfiguration(ActionConfig(), self.det_label_schema) + _model = ModelEntity(self.det_dataset, _config) + resultset = ResultSetEntity(_model, self.det_dataset, self.det_dataset) + self.det_task.evaluate(resultset) + assert resultset.performance.score.value == 0.0 + + @pytest.mark.parametrize("precision", [ModelPrecision.FP16, ModelPrecision.FP32]) + @e2e_pytest_unit + def test_export(self, mocker, precision: ModelPrecision) -> None: + """Test export function. + + + 1. Create model entity + 2. Run export function + 3. Check output model attributes + """ + _config = ModelConfiguration(ActionConfig(), self.cls_label_schema) + _model = ModelEntity(self.cls_dataset, _config) + mocker.patch("otx.algorithms.action.adapters.mmaction.task.Exporter", return_value=MockExporter(self.cls_task)) + mocker.patch("torch.load", return_value={}) + mocker.patch("torch.nn.Module.load_state_dict", return_value=True) + + self.cls_task.export(ExportType.OPENVINO, _model, precision, False) + + assert _model.model_format == ModelFormat.OPENVINO + assert _model.optimization_type == ModelOptimizationType.MO + assert _model.precision[0] == precision + assert _model.get_data("openvino.bin") is not None + assert _model.get_data("openvino.xml") is not None + assert _model.get_data("confidence_threshold") is not None + assert _model.precision == self.cls_task._precision + assert _model.optimization_methods == self.cls_task._optimization_methods + assert _model.get_data("label_schema.json") is not None diff --git a/tests/unit/algorithms/action/adapters/mmaction/utils/test_action_config_utils.py b/tests/unit/algorithms/action/adapters/mmaction/utils/test_action_config_utils.py deleted file mode 100644 index 3aad5a00fd0..00000000000 --- a/tests/unit/algorithms/action/adapters/mmaction/utils/test_action_config_utils.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Unit Test for otx.algorithms.action.adapters.mmaction.utils.config_utils.""" - -# Copyright (C) 2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -import tempfile - -import pytest -from mmcv.utils import Config - -from otx.algorithms.action.adapters.mmaction.utils import ( - patch_config, - prepare_for_training, - set_data_classes, -) -from otx.algorithms.common.adapters.mmcv.utils import get_data_cfg -from otx.api.entities.annotation import NullAnnotationSceneEntity -from otx.api.entities.dataset_item import DatasetItemEntity -from otx.api.entities.datasets import DatasetEntity -from otx.api.entities.image import Image -from otx.api.entities.label import Domain, LabelEntity -from otx.api.entities.model_template import TaskType -from tests.test_suite.e2e_test_system import e2e_pytest_unit - -CLS_CONFIG_NAME = "otx/algorithms/action/configs/classification/x3d/model.py" -DET_CONFIG_NAME = "otx/algorithms/action/configs/detection/x3d_fast_rcnn/model.py" -CLS_CONFIG = Config.fromfile(CLS_CONFIG_NAME) -DET_CONFIG = Config.fromfile(DET_CONFIG_NAME) - - -@e2e_pytest_unit -def test_patch_config() -> None: - """Test patch_config function. - - - 1. Check error when gives wrong task type - 2. Check work_dir - 3. Check merged data pipeline - 4. Check dataset type - """ - - cls_datapipeline_path = "otx/algorithms/action/configs/classification/x3d/data_pipeline.py" - with tempfile.TemporaryDirectory() as work_dir: - with pytest.raises(NotImplementedError): - patch_config(CLS_CONFIG, cls_datapipeline_path, work_dir, TaskType.CLASSIFICATION) - - patch_config(CLS_CONFIG, cls_datapipeline_path, work_dir, TaskType.ACTION_CLASSIFICATION) - assert CLS_CONFIG.work_dir == work_dir - assert CLS_CONFIG.get("train_pipeline", None) - for subset in ("train", "val", "test", "unlabeled"): - cfg = CLS_CONFIG.data.get(subset, None) - if not cfg: - continue - assert cfg.type == "OTXActionClsDataset" - - det_datapipeline_path = "otx/algorithms/action/configs/detection/x3d_fast_rcnn/data_pipeline.py" - patch_config(DET_CONFIG, det_datapipeline_path, work_dir, TaskType.ACTION_DETECTION) - assert DET_CONFIG.work_dir == work_dir - assert DET_CONFIG.get("train_pipeline", None) - for subset in ("train", "val", "test", "unlabeled"): - cfg = DET_CONFIG.data.get(subset, None) - if not cfg: - continue - assert cfg.type == "OTXActionDetDataset" - - -@e2e_pytest_unit -def test_set_data_classes() -> None: - """Test set_data_classes funciton. - - - 1. Create sample labels - 2. Check classification label length is appropriate - """ - - labels = [ - LabelEntity(name="0", domain=Domain.ACTION_CLASSIFICATION), - LabelEntity(name="1", domain=Domain.ACTION_CLASSIFICATION), - LabelEntity(name="2", domain=Domain.ACTION_CLASSIFICATION), - ] - set_data_classes(CLS_CONFIG, labels, TaskType.ACTION_CLASSIFICATION) - assert CLS_CONFIG.model["cls_head"].num_classes == len(labels) - - labels = [ - LabelEntity(name="0", domain=Domain.ACTION_DETECTION), - LabelEntity(name="1", domain=Domain.ACTION_DETECTION), - LabelEntity(name="2", domain=Domain.ACTION_DETECTION), - ] - set_data_classes(DET_CONFIG, labels, TaskType.ACTION_DETECTION) - assert DET_CONFIG.model["roi_head"]["bbox_head"].num_classes == len(labels) + 1 - assert DET_CONFIG.model["roi_head"]["bbox_head"]["topk"] == len(labels) - 1 - - -@e2e_pytest_unit -def test_prepare_for_training() -> None: - """Test prepare_for_training function. - - - 1. Create sample DatasetEntity - 2. Check config.data.train - 3. Check config.data.val - """ - - item = DatasetItemEntity(media=Image(file_path="iamge.jpg"), annotation_scene=NullAnnotationSceneEntity()) - dataset = DatasetEntity(items=[item]) - - CLS_CONFIG.runner = {} - prepare_for_training(CLS_CONFIG, dataset, dataset) - - assert get_data_cfg(CLS_CONFIG, "train").otx_dataset == dataset - assert get_data_cfg(CLS_CONFIG, "val").otx_dataset == dataset diff --git a/tests/unit/algorithms/action/tasks/test_action_openvino.py b/tests/unit/algorithms/action/adapters/openvino/test_task.py similarity index 82% rename from tests/unit/algorithms/action/tasks/test_action_openvino.py rename to tests/unit/algorithms/action/adapters/openvino/test_task.py index 65b343ffa15..bf98b37016b 100644 --- a/tests/unit/algorithms/action/tasks/test_action_openvino.py +++ b/tests/unit/algorithms/action/adapters/openvino/test_task.py @@ -1,4 +1,4 @@ -"""Unit Test for otx.algorithms.action.tasks.openvino.""" +"""Unit Test for otx.algorithms.action.adapters.openvino.task.""" # Copyright (C) 2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 @@ -13,7 +13,7 @@ from otx.algorithms.action.adapters.openvino import ActionOVClsDataLoader from otx.algorithms.action.configs.base.configuration import ActionConfig -from otx.algorithms.action.tasks.openvino import ( +from otx.algorithms.action.adapters.openvino.task import ( ActionOpenVINOInferencer, ActionOpenVINOTask, DataLoaderWrapper, @@ -135,8 +135,8 @@ def setup(self, mocker) -> None: group_type=LabelGroupType.EXCLUSIVE, ) self.label_schema.add_group(label_group) - mocker.patch("otx.algorithms.action.tasks.openvino.OpenvinoAdapter.__init__", return_value=None) - mocker.patch("otx.algorithms.action.tasks.openvino.Model.create_model", return_value=MockModel()) + mocker.patch("otx.algorithms.action.adapters.openvino.task.OpenvinoAdapter.__init__", return_value=None) + mocker.patch("otx.algorithms.action.adapters.openvino.task.Model.create_model", return_value=MockModel()) self.inferencer = ActionOpenVINOInferencer( "ACTION_CLASSIFICATION", ActionConfig(), @@ -181,7 +181,7 @@ def test_pre_process(self) -> None: def test_post_process(self, mocker) -> None: """Test post_process function.""" mocker.patch( - "otx.algorithms.action.tasks.openvino.ClassificationToAnnotationConverter.convert_to_annotation", + "otx.algorithms.action.adapters.openvino.task.ClassificationToAnnotationConverter.convert_to_annotation", side_effect=return_args, ) assert ( @@ -200,14 +200,16 @@ def test_predict(self, mocker) -> None: """Test predict function.""" mocker.patch( - "otx.algorithms.action.tasks.openvino.ActionOpenVINOInferencer.pre_process", + "otx.algorithms.action.adapters.openvino.task.ActionOpenVINOInferencer.pre_process", return_value=("data", "metadata"), ) mocker.patch( - "otx.algorithms.action.tasks.openvino.ActionOpenVINOInferencer.forward", return_value="raw_predictions" + "otx.algorithms.action.adapters.openvino.task.ActionOpenVINOInferencer.forward", + return_value="raw_predictions", ) mocker.patch( - "otx.algorithms.action.tasks.openvino.ActionOpenVINOInferencer.post_process", return_value="predictions" + "otx.algorithms.action.adapters.openvino.task.ActionOpenVINOInferencer.post_process", + return_value="predictions", ) dataset = generate_action_cls_otx_dataset(1, 10, self.labels) @@ -257,7 +259,9 @@ def setup(self, mocker) -> None: def test_load_inferencer(self, mocker) -> None: """Test load_inferencer function.""" - mocker.patch("otx.algorithms.action.tasks.openvino.ActionOpenVINOInferencer", return_value=MockOVInferencer()) + mocker.patch( + "otx.algorithms.action.adapters.openvino.task.ActionOpenVINOInferencer", return_value=MockOVInferencer() + ) task = ActionOpenVINOTask(self.task_environment) assert isinstance(task.inferencer, MockOVInferencer) @@ -270,9 +274,12 @@ def test_infer(self, mocker) -> None: """Test infer function.""" mocker.patch( - "otx.algorithms.action.tasks.openvino.ActionOpenVINOTask.load_inferencer", return_value=MockOVInferencer() + "otx.algorithms.action.adapters.openvino.task.ActionOpenVINOTask.load_inferencer", + return_value=MockOVInferencer(), + ) + mocker.patch( + "otx.algorithms.action.adapters.openvino.task.get_ovdataloader", return_value=MockDataloader(self.dataset) ) - mocker.patch("otx.algorithms.action.tasks.openvino.get_ovdataloader", return_value=MockDataloader(self.dataset)) task = ActionOpenVINOTask(self.task_environment) output = task.infer(self.dataset.with_empty_annotations()) assert output[0].annotation_scene.kind == AnnotationSceneKind.PREDICTION @@ -286,7 +293,8 @@ def get_performance(self): return 1.0 mocker.patch( - "otx.algorithms.action.tasks.openvino.ActionOpenVINOTask.load_inferencer", return_value=MockOVInferencer() + "otx.algorithms.action.adapters.openvino.task.ActionOpenVINOTask.load_inferencer", + return_value=MockOVInferencer(), ) task = ActionOpenVINOTask(self.task_environment) @@ -311,7 +319,8 @@ def test_deploy(self, mocker) -> None: """Test function for deploy function.""" mocker.patch( - "otx.algorithms.action.tasks.openvino.ActionOpenVINOTask.load_inferencer", return_value=MockOVInferencer() + "otx.algorithms.action.adapters.openvino.task.ActionOpenVINOTask.load_inferencer", + return_value=MockOVInferencer(), ) task = ActionOpenVINOTask(self.task_environment) assert self.model.exportable_code is None @@ -335,18 +344,21 @@ def mock_save_model(model, tempdir, model_name): with open(os.path.join(tempdir, "model.bin"), "wb") as f: f.write(np.ndarray(1).tobytes()) - mocker.patch("otx.algorithms.action.tasks.openvino.get_ovdataloader", return_value=MockDataloader(self.dataset)) mocker.patch( - "otx.algorithms.action.tasks.openvino.DataLoaderWrapper", return_value=MockDataloader(self.dataset) + "otx.algorithms.action.adapters.openvino.task.get_ovdataloader", return_value=MockDataloader(self.dataset) + ) + mocker.patch( + "otx.algorithms.action.adapters.openvino.task.DataLoaderWrapper", return_value=MockDataloader(self.dataset) ) - mocker.patch("otx.algorithms.action.tasks.openvino.load_model", return_value=self.model) - mocker.patch("otx.algorithms.action.tasks.openvino.get_nodes_by_type", return_value=False) - mocker.patch("otx.algorithms.action.tasks.openvino.IEEngine", return_value=True) - mocker.patch("otx.algorithms.action.tasks.openvino.create_pipeline", return_value=MockPipeline()) - mocker.patch("otx.algorithms.action.tasks.openvino.compress_model_weights", return_value=True) - mocker.patch("otx.algorithms.action.tasks.openvino.save_model", side_effect=mock_save_model) + mocker.patch("otx.algorithms.action.adapters.openvino.task.load_model", return_value=self.model) + mocker.patch("otx.algorithms.action.adapters.openvino.task.get_nodes_by_type", return_value=False) + mocker.patch("otx.algorithms.action.adapters.openvino.task.IEEngine", return_value=True) + mocker.patch("otx.algorithms.action.adapters.openvino.task.create_pipeline", return_value=MockPipeline()) + mocker.patch("otx.algorithms.action.adapters.openvino.task.compress_model_weights", return_value=True) + mocker.patch("otx.algorithms.action.adapters.openvino.task.save_model", side_effect=mock_save_model) mocker.patch( - "otx.algorithms.action.tasks.openvino.ActionOpenVINOTask.load_inferencer", return_value=MockOVInferencer() + "otx.algorithms.action.adapters.openvino.task.ActionOpenVINOTask.load_inferencer", + return_value=MockOVInferencer(), ) task = ActionOpenVINOTask(self.task_environment) task.optimize(OptimizationType.POT, self.dataset, self.model, OptimizationParameters()) diff --git a/tests/unit/algorithms/action/tasks/__init__.py b/tests/unit/algorithms/action/tasks/__init__.py deleted file mode 100644 index 79931efa777..00000000000 --- a/tests/unit/algorithms/action/tasks/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright (C) 2021-2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# diff --git a/tests/unit/algorithms/action/tasks/test_action_inference.py b/tests/unit/algorithms/action/tasks/test_action_inference.py deleted file mode 100644 index 2d82f3cc72b..00000000000 --- a/tests/unit/algorithms/action/tasks/test_action_inference.py +++ /dev/null @@ -1,411 +0,0 @@ -"""Unit Test for otx.algorithms.action.tasks.inference.""" - -# Copyright (C) 2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -import os -from typing import Any, Dict - -import numpy as np -import pytest -import torch -from mmcv.utils import Config -from torch import nn - -from otx.algorithms.action.configs.base.configuration import ActionConfig -from otx.algorithms.action.tasks.inference import ActionInferenceTask -from otx.api.configuration import ConfigurableParameters -from otx.api.entities.dataset_item import DatasetItemEntity -from otx.api.entities.datasets import DatasetEntity -from otx.api.entities.inference_parameters import InferenceParameters -from otx.api.entities.label import Domain -from otx.api.entities.label_schema import LabelGroup, LabelGroupType, LabelSchemaEntity -from otx.api.entities.model import ( - ModelConfiguration, - ModelEntity, - ModelFormat, - ModelOptimizationType, - ModelPrecision, -) -from otx.api.entities.model_template import InstantiationType, TaskFamily, TaskType -from otx.api.entities.resultset import ResultSetEntity -from otx.api.entities.task_environment import TaskEnvironment -from otx.api.usecases.tasks.interfaces.export_interface import ExportType -from tests.test_suite.e2e_test_system import e2e_pytest_unit -from tests.unit.algorithms.action.test_helpers import ( - MockModelTemplate, - generate_action_cls_otx_dataset, - generate_action_det_otx_dataset, - generate_labels, - return_inputs, -) - - -class MockModule(nn.Module): - """Mock class for nn.Module.""" - - def forward(self, inputs: Any): - return inputs - - -class MockModel(nn.Module): - """Mock class for pytorch model.""" - - def __init__(self, task_type): - super().__init__() - self.module = MockModule() - self.module.backbone = MockModule() - self.backbone = MockModule() - self.task_type = task_type - - def forward(self, return_loss: bool, imgs: DatasetItemEntity): - forward_hooks = list(self.module.backbone._forward_hooks.values()) - for hook in forward_hooks: - hook(1, 2, 3) - if self.task_type == "cls": - return np.array([[0, 0, 1]]) - return [[np.array([[0, 0, 1, 1, 0.1]]), np.array([[0, 0, 1, 1, 0.2]]), np.array([[0, 0, 1, 1, 0.7]])]] - - @staticmethod - def named_parameters(): - return {"name": torch.Tensor([0.5])}.items() - - -class MockDataset(DatasetEntity): - """Mock class for mm_dataset.""" - - def __init__(self, dataset: DatasetEntity, task_type: str): - self.dataset = dataset - self.task_type = task_type - - def evaluate(self, prediction, *args, **kwargs): - if self.task_type == "cls": - return {"accuracy": 1.0} - else: - return {"mAP@0.5IOU": 1.0} - - -class MockDataLoader: - """Mock class for data loader.""" - - def __init__(self, dataset: DatasetEntity): - self.dataset = dataset - self.iter = iter(self.dataset) - - def __len__(self) -> int: - return len(self.dataset) - - def __next__(self) -> Dict[str, DatasetItemEntity]: - return {"imgs": next(self.iter)} - - def __iter__(self): - return self - - -class MockExporter: - """Mock class for Exporter.""" - - def __init__(self, recipe_cfg, weights, deploy_cfg, work_dir, half_precision): - self.work_dir = work_dir - - def export(self): - dummy_data = np.ndarray((1, 1, 1)) - with open(self.work_dir + ".bin", "wb") as f: - f.write(dummy_data) - with open(self.work_dir + ".xml", "wb") as f: - f.write(dummy_data) - with open(self.work_dir + ".onnx", "wb") as f: - f.write(dummy_data) - - -class TestActionInferenceTask: - """Test class for ActionInferenceTask. - - Details are explained in each test function. - """ - - @pytest.fixture(autouse=True) - def setup(self) -> None: - self.video_len = 3 - self.frame_len = 3 - - cls_labels = generate_labels(3, Domain.ACTION_CLASSIFICATION) - self.cls_label_schema = LabelSchemaEntity() - cls_label_group = LabelGroup( - name="labels", - labels=cls_labels, - group_type=LabelGroupType.EXCLUSIVE, - ) - self.cls_label_schema.add_group(cls_label_group) - cls_template = MockModelTemplate( - model_template_id="template_id", - model_template_path="template_path", - name="template", - task_family=TaskFamily.VISION, - task_type=TaskType.ACTION_CLASSIFICATION, - instantiation=InstantiationType.CLASS, - ) - self.cls_task_environment = TaskEnvironment( - model=None, - hyper_parameters=ConfigurableParameters(header="h-params"), - label_schema=self.cls_label_schema, - model_template=cls_template, - ) - - self.cls_dataset = generate_action_cls_otx_dataset(self.video_len, self.frame_len, cls_labels) - self.cls_task = ActionInferenceTask(task_environment=self.cls_task_environment) - - det_labels = generate_labels(3, Domain.ACTION_DETECTION) - self.det_label_schema = LabelSchemaEntity() - det_label_group = LabelGroup( - name="labels", - labels=det_labels, - group_type=LabelGroupType.EXCLUSIVE, - ) - self.det_label_schema.add_group(det_label_group) - det_template = MockModelTemplate( - model_template_id="template_id", - model_template_path="template_path", - name="template", - task_family=TaskFamily.VISION, - task_type=TaskType.ACTION_DETECTION, - instantiation=InstantiationType.CLASS, - ) - self.det_task_environment = TaskEnvironment( - model=None, - hyper_parameters=ConfigurableParameters(header="h-params"), - label_schema=self.det_label_schema, - model_template=det_template, - ) - - self.det_dataset = generate_action_det_otx_dataset(self.video_len, self.frame_len, det_labels)[0] - self.det_task = ActionInferenceTask(task_environment=self.det_task_environment) - - @e2e_pytest_unit - # TODO Sepearate add prediction function test and infer funciton test - def test_infer(self, mocker) -> None: - """Test infer function. - - - 1. Create mock model for action classification - 2. Create mock recipe for action classification - 3. Run infer funciton - 4. Check whether inference results are added to output - 5. Do 1 - 4 for action detection - """ - - mocker.patch( - "otx.algorithms.action.tasks.inference.build_dataset", return_value=MockDataset(self.cls_dataset, "cls") - ) - mocker.patch( - "otx.algorithms.action.tasks.inference.build_dataloader", return_value=MockDataLoader(self.cls_dataset) - ) - mocker.patch("otx.algorithms.action.tasks.inference.MMDataParallel", return_value=MockModel("cls")) - self.cls_task._model = MockModel("cls") - self.cls_task._recipe_cfg = Config( - { - "data": {"test": {"otx_dataset": None}, "workers_per_gpu": 1}, - "gpu_ids": [0], - "evaluation": {"final_metric": "accuracy"}, - } - ) - inference_parameters = InferenceParameters(is_evaluation=True) - outputs = self.cls_task.infer(self.cls_dataset, inference_parameters) - for output in outputs: - assert len(output.get_annotations()[0].get_labels()) == 2 - - mocker.patch( - "otx.algorithms.action.tasks.inference.build_dataset", return_value=MockDataset(self.det_dataset, "det") - ) - mocker.patch( - "otx.algorithms.action.tasks.inference.build_dataloader", return_value=MockDataLoader(self.det_dataset) - ) - mocker.patch("otx.algorithms.action.tasks.inference.MMDataParallel", return_value=MockModel("det")) - self.det_task._model = MockModel("det") - self.det_task._recipe_cfg = Config( - { - "data": {"test": {"otx_dataset": None}, "workers_per_gpu": 1}, - "gpu_ids": [0], - "evaluation": {"final_metric": "mAP@0.5IOU"}, - } - ) - inference_parameters = InferenceParameters(is_evaluation=True) - outputs = self.det_task.infer(self.det_dataset, inference_parameters) - for output in outputs: - assert len(output.get_annotations()) == 4 - - @e2e_pytest_unit - def test_evaluate(self) -> None: - """Test evaluate function. - - - 1. Create model entity - 2. Create result set entity - 3. Run evaluate function with same dataset, this should give 100% accuracy - 4. Run evaluate function with empty dataset, this should give 0% accuracy - 5. Do 1 - 4 for action detection - """ - _config = ModelConfiguration(ActionConfig(), self.cls_label_schema) - _model = ModelEntity(self.cls_dataset, _config) - resultset = ResultSetEntity(_model, self.cls_dataset, self.cls_dataset) - self.cls_task.evaluate(resultset) - assert resultset.performance.score.value == 1.0 - - resultset = ResultSetEntity(_model, self.cls_dataset, self.cls_dataset.with_empty_annotations()) - self.cls_task.evaluate(resultset) - assert resultset.performance.score.value == 0.0 - - _config = ModelConfiguration(ActionConfig(), self.det_label_schema) - _model = ModelEntity(self.det_dataset, _config) - resultset = ResultSetEntity(_model, self.det_dataset, self.det_dataset) - self.det_task.evaluate(resultset) - assert resultset.performance.score.value == 0.0 - - @e2e_pytest_unit - def test_initialize_post_hook(self) -> None: - """Test _initialize_post_hook funciton.""" - - options = None - assert self.cls_task._initialize_post_hook(options) is None - - options = {"deploy_cfg": Config()} - self.cls_task._initialize_post_hook(options) - assert isinstance(self.cls_task.deploy_cfg, Config) - - @pytest.mark.parametrize("precision", [ModelPrecision.FP16, ModelPrecision.FP32]) - @pytest.mark.parametrize("dump_features", [False]) - @e2e_pytest_unit - def test_export(self, mocker, precision: ModelPrecision, dump_features: bool) -> None: - """Test export function. - - - 1. Create model entity - 2. Run export function - 3. Check output model attributes - """ - _config = ModelConfiguration(ActionConfig(), self.cls_label_schema) - _model = ModelEntity(self.cls_dataset, _config) - self.cls_task._model = nn.Module() - self.cls_task._recipe_cfg = None - self.cls_task.deploy_cfg = Config( - dict(codebase_config=dict(type="mmdet", task="ObjectDetection"), backend_config=dict(type="openvino")) - ) - mocker.patch("otx.algorithms.action.tasks.inference.ActionInferenceTask._init_task", return_value=True) - mocker.patch("otx.algorithms.action.tasks.inference.Exporter", side_effect=MockExporter) - self.cls_task.export(ExportType.OPENVINO, _model, precision, dump_features) - - assert _model.model_format == ModelFormat.OPENVINO - assert _model.optimization_type == ModelOptimizationType.MO - assert _model.precision[0] == precision - assert _model.get_data("openvino.bin") is not None - assert _model.get_data("openvino.xml") is not None - assert _model.get_data("model.onnx") is not None - assert _model.get_data("confidence_threshold") is not None - assert _model.precision == self.cls_task._precision - assert _model.optimization_methods == self.cls_task._optimization_methods - assert _model.get_data("label_schema.json") is not None - assert _model.has_xai == dump_features - - @e2e_pytest_unit - def test_init_task(self, mocker) -> None: - """Test _init_task function. - - Check model is generated from _init_task function. - """ - mocker.patch("otx.algorithms.action.tasks.inference.ActionInferenceTask._initialize", return_value=True) - mocker.patch( - "otx.algorithms.action.tasks.inference.ActionInferenceTask._load_model", return_value=MockModel("cls") - ) - with pytest.raises(RuntimeError): - self.cls_task._init_task() - - self.cls_task._recipe_cfg = Config() - self.cls_task._init_task() - assert isinstance(self.cls_task._model, MockModel) - - @e2e_pytest_unit - def test_load_model(self, mocker) -> None: - """Test _load_model function. - - Check _load_model function can run _create_model function under various situations - """ - - mocker.patch( - "otx.algorithms.action.tasks.inference.ActionInferenceTask._create_model", return_value=MockModel("cls") - ) - mocker.patch("otx.algorithms.action.tasks.inference.load_state_dict", return_value=True) - mocker.patch( - "otx.algorithms.action.tasks.inference.torch.load", - return_value={"confidence_threshold": 0.01, "model": np.array([0.01])}, - ) - - with pytest.raises(Exception): - self.cls_task._load_model(None) - - _config = ModelConfiguration(ActionConfig(), self.cls_label_schema) - _model_entity = ModelEntity(self.cls_dataset, _config) - _model_entity.set_data("weights.pth", np.ndarray((1, 1, 1)).tobytes()) - - self.cls_task._recipe_cfg = Config({"load_from": "weights.pth"}) - model = self.cls_task._load_model(_model_entity) - assert isinstance(model, MockModel) - - model = self.cls_task._load_model(None) - assert isinstance(model, MockModel) - - def test_create_model(self, mocker) -> None: - """Test _create_model function. - - Check _create_model function can run build_model funciton under various situations - """ - mocker.patch("otx.algorithms.action.tasks.inference.build_model", return_value=MockModel("cls")) - mocker.patch("otx.algorithms.action.tasks.inference.load_checkpoint", return_value=True) - - _config = Config({"model": Config(), "load_from": "weights.pth"}) - model = self.cls_task._create_model(_config, False) - assert isinstance(model, MockModel) - model = self.cls_task._create_model(_config, True) - assert isinstance(model, MockModel) - - def test_unload(self) -> None: - """Test unload function.""" - self.cls_task.unload() - - def test_init_recipe_hparam(self, mocker) -> None: - """Test _init_recipe_hparam function.""" - - mocker.patch( - "otx.algorithms.action.tasks.inference.BaseTask._init_recipe_hparam", - return_value=Config( - {"data": {"samples_per_gpu": 4}, "lr_config": {"warmup_iters": 3}, "runner": {"max_epochs": 10}} - ), - ) - - self.cls_task._recipe_cfg = Config({"lr_config": Config()}) - out = self.cls_task._init_recipe_hparam() - - assert self.cls_task._recipe_cfg.lr_config.warmup == "linear" - assert self.cls_task._recipe_cfg.lr_config.warmup_by_epoch is True - assert self.cls_task._recipe_cfg.total_epochs == 10 - assert out.data.videos_per_gpu == 4 - assert out.use_adaptive_interval == self.cls_task._hyperparams.learning_parameters.use_adaptive_interval - - def test_init_recipe(self, mocker) -> None: - """Test _init_recipe funciton.""" - - mocker.patch("otx.algorithms.action.tasks.inference.Config.fromfile", side_effect=return_inputs) - mocker.patch("otx.algorithms.action.tasks.inference.patch_config", return_value=True) - mocker.patch("otx.algorithms.action.tasks.inference.set_data_classes", return_value=True) - - self.cls_task._init_recipe() - recipe_root = os.path.abspath(os.path.dirname(self.cls_task.template_file_path)) - assert self.cls_task._recipe_cfg == os.path.join(recipe_root, "model.py") - - def test_init_model_cfg(self, mocker) -> None: - """Test _init_model_cfg function.""" - - mocker.patch("otx.algorithms.action.tasks.inference.Config.fromfile", side_effect=return_inputs) - - model_cfg = self.cls_task._init_model_cfg() - assert model_cfg == os.path.join(self.cls_task._model_dir, "model.py") diff --git a/tests/unit/algorithms/action/tasks/test_action_train.py b/tests/unit/algorithms/action/tasks/test_action_train.py deleted file mode 100644 index 4836b49d6c9..00000000000 --- a/tests/unit/algorithms/action/tasks/test_action_train.py +++ /dev/null @@ -1,204 +0,0 @@ -"""Unit Test for otx.algorithms.action.tasks.train.""" - -# Copyright (C) 2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -import os - -import pytest -import torch -from mmcv.utils import Config - -from otx.algorithms.action.configs.base.configuration import ActionConfig -from otx.algorithms.action.tasks.train import ActionTrainTask -from otx.api.configuration import ConfigurableParameters -from otx.api.entities.label import Domain -from otx.api.entities.label_schema import LabelGroup, LabelGroupType, LabelSchemaEntity -from otx.api.entities.metrics import ( - BarMetricsGroup, - LineMetricsGroup, - Performance, - ScoreMetric, -) -from otx.api.entities.model import ModelConfiguration, ModelEntity -from otx.api.entities.model_template import InstantiationType, TaskFamily, TaskType -from otx.api.entities.task_environment import TaskEnvironment -from tests.test_suite.e2e_test_system import e2e_pytest_unit -from tests.unit.algorithms.action.test_helpers import ( - MockModelTemplate, - generate_action_cls_otx_dataset, - generate_labels, -) - - -class TestActionTrainTask: - """Test class for ActionTrainTask class.""" - - @pytest.fixture(autouse=True) - def setup(self) -> None: - self.video_len = 3 - self.frame_len = 3 - - cls_labels = generate_labels(3, Domain.ACTION_CLASSIFICATION) - self.cls_label_schema = LabelSchemaEntity() - cls_label_group = LabelGroup( - name="labels", - labels=cls_labels, - group_type=LabelGroupType.EXCLUSIVE, - ) - self.cls_label_schema.add_group(cls_label_group) - cls_template = MockModelTemplate( - model_template_id="template_id", - model_template_path="template_path", - name="template", - task_family=TaskFamily.VISION, - task_type=TaskType.ACTION_CLASSIFICATION, - instantiation=InstantiationType.CLASS, - ) - self.cls_task_environment = TaskEnvironment( - model=None, - hyper_parameters=ConfigurableParameters(header="h-params"), - label_schema=self.cls_label_schema, - model_template=cls_template, - ) - - self.cls_dataset = generate_action_cls_otx_dataset(self.video_len, self.frame_len, cls_labels) - self.cls_task = ActionTrainTask(task_environment=self.cls_task_environment) - - @e2e_pytest_unit - def test_save_model(self, mocker) -> None: - """Test save_model function.""" - - mocker.patch("otx.algorithms.action.tasks.train.torch.load", return_value={"state_dict": None}) - - config = ModelConfiguration(ActionConfig(), self.cls_label_schema) - output_model = ModelEntity(self.cls_dataset, config) - self.cls_task.save_model(output_model) - assert output_model.get_data("weights.pth") is not None - assert output_model.get_data("label_schema.json") is not None - assert output_model.precision == self.cls_task._precision - - @e2e_pytest_unit - def test_cancel_training(self) -> None: - """Test cance_trainng function.""" - - class _MockCanceInterface: - def cancel(self): - raise RuntimeError("Checking for calling this function") - - self.cls_task.cancel_training() - assert self.cls_task.reserved_cancel is True - - self.cls_task.cancel_interface = _MockCanceInterface() - with pytest.raises(RuntimeError): - self.cls_task.cancel_training() - - @e2e_pytest_unit - def test_train(self, mocker) -> None: - """Test train function.""" - - config = ModelConfiguration(ActionConfig(), self.cls_label_schema) - output_model = ModelEntity(self.cls_dataset, config) - self.cls_task._should_stop = True - self.cls_task.train(self.cls_dataset, output_model) - assert self.cls_task._should_stop is False - assert self.cls_task._is_training is False - - mocker.patch("otx.algorithms.action.tasks.train.ActionTrainTask._init_task", return_value=True) - mocker.patch("otx.algorithms.action.tasks.train.ActionTrainTask._train_model", return_value=True) - mocker.patch("otx.algorithms.action.tasks.train.ActionTrainTask._get_output_model", return_value=True) - mocker.patch("otx.algorithms.action.tasks.train.ActionTrainTask._get_final_eval_results", return_value=0.5) - mocker.patch("otx.algorithms.action.tasks.train.ActionTrainTask.save_model", return_value=True) - - self.cls_task._recipe_cfg = None - self.cls_task._should_stop = False - with pytest.raises(Exception): - self.cls_task.train(self.cls_dataset, output_model) - self.cls_task._recipe_cfg = Config() - self.cls_task.train(self.cls_dataset, output_model) - assert output_model.performance == 0.5 - assert self.cls_task._is_training is False - - @e2e_pytest_unit - def test_train_model(self, mocker) -> None: - """Test _train_model function.""" - - with pytest.raises(Exception): - self.cls_task._train_model(self.cls_dataset) - - self.cls_task._recipe_cfg = Config({"work_dir": self.cls_task._output_path}) - self.cls_task._model = torch.nn.Module() - - def _mock_train_model(*args, **kwargs): - with open(os.path.join(self.cls_task._recipe_cfg.work_dir, "best.pth"), "wb") as f: - torch.save(torch.randn(1), f) - - mocker.patch( - "otx.algorithms.action.tasks.train.prepare_for_training", return_value=Config({"data": {"train": None}}) - ) - mocker.patch("otx.algorithms.action.tasks.train.build_dataset", return_value=True) - mocker.patch("otx.algorithms.action.tasks.train.train_model", side_effect=_mock_train_model) - - out = self.cls_task._train_model(self.cls_dataset) - assert out["final_ckpt"] is not None - assert out["final_ckpt"].split("/")[-1] == "best.pth" - - @e2e_pytest_unit - def test_get_output_model(self, mocker) -> None: - """Test _get_output_model function.""" - - self.cls_task._model = torch.nn.Module() - sample_results = {"final_ckpt": None} - self.cls_task._get_output_model(sample_results) - - mocker.patch("otx.algorithms.action.tasks.train.torch.load", return_value={"state_dict": {}}) - sample_results = {"final_ckpt": "checkpoint_file_path"} - self.cls_task._get_output_model(sample_results) - - @e2e_pytest_unit - def test_get_final_eval_results(self, mocker) -> None: - """Test _get_final_eval_results.""" - - class _mock_metric: - def __init__(self): - self.performance = Performance(ScoreMetric("accuracy", 1.0)) - - def get_performance(self): - return self.performance - - config = ModelConfiguration(ActionConfig(), self.cls_label_schema) - output_model = ModelEntity(self.cls_dataset, config) - - mocker.patch("otx.algorithms.action.tasks.train.ActionTrainTask._infer_model", return_value=(True, True)) - mocker.patch("otx.algorithms.action.tasks.train.ActionTrainTask._add_predictions_to_dataset", return_value=True) - mocker.patch( - "otx.algorithms.action.tasks.train.ActionTrainTask._add_det_predictions_to_dataset", return_value=True - ) - mocker.patch("otx.algorithms.action.tasks.train.ActionTrainTask._generate_training_metrics", return_value=[]) - - mocker.patch("otx.algorithms.action.tasks.train.ActionTrainTask._get_metric", return_value=_mock_metric()) - self.cls_task._recipe_cfg = Config({"evaluation": {"final_metric": "accuracy"}}) - performance = self.cls_task._get_final_eval_results(self.cls_dataset, output_model) - assert performance.score.name == "accuracy" - assert performance.score.value == 1.0 - - self.cls_task._task_type = TaskType.ACTION_DETECTION - performance = self.cls_task._get_final_eval_results(self.cls_dataset, output_model) - - @e2e_pytest_unit - def test_generate_training_metrics(self) -> None: - """Test _generate_training_metrics fucntion.""" - - class MockCurve: - def __init__(self, x: list, y: list): - self.x = x - self.y = y - - sample_learning_curve = { - "dummy0": MockCurve([0, 1, 2], [2, 1, 0]), - "dummy1": MockCurve([0, 1, 2], [2, 1]), - } - output = self.cls_task._generate_training_metrics(sample_learning_curve, 1.0, "accuracy") - assert isinstance(output[0], LineMetricsGroup) - assert isinstance(output[-1], BarMetricsGroup) diff --git a/tests/unit/algorithms/action/test_helpers.py b/tests/unit/algorithms/action/test_helpers.py index 9b896987e37..834da3d82b2 100644 --- a/tests/unit/algorithms/action/test_helpers.py +++ b/tests/unit/algorithms/action/test_helpers.py @@ -22,6 +22,8 @@ from otx.api.entities.model_template import ModelTemplate from otx.api.entities.scored_label import ScoredLabel from otx.api.entities.shapes.rectangle import Rectangle +from otx.api.entities.subset import Subset +from otx.api.entities.task_environment import TaskEnvironment class MockImage(Image): @@ -57,6 +59,10 @@ def generate_action_cls_otx_dataset(video_len: int, frame_len: int, labels: List items: List[DatasetItemEntity] = [] for video_id in range(video_len): + if video_id > 1: + subset = Subset.VALIDATION + else: + subset = Subset.TRAINING for frame_idx in range(frame_len): item = DatasetItemEntity( media=MockImage(f"{video_id}_{frame_idx}.png"), @@ -65,6 +71,7 @@ def generate_action_cls_otx_dataset(video_len: int, frame_len: int, labels: List kind=AnnotationSceneKind.ANNOTATION, ), metadata=[MetadataItemEntity(data=VideoMetadata(video_id, frame_idx, is_empty_frame=False))], + subset=subset, ) items.append(item) dataset = DatasetEntity(items=items) @@ -77,6 +84,10 @@ def generate_action_det_otx_dataset(video_len: int, frame_len: int, labels: List items: List[DatasetItemEntity] = [] proposals: Dict[str, List[float]] = {} for video_id in range(video_len): + if video_id > 1: + subset = Subset.VALIDATION + else: + subset = Subset.TRAINING for frame_idx in range(frame_len): if frame_idx % 2 == 0: item = DatasetItemEntity( @@ -86,6 +97,7 @@ def generate_action_det_otx_dataset(video_len: int, frame_len: int, labels: List kind=AnnotationSceneKind.ANNOTATION, ), metadata=[MetadataItemEntity(data=VideoMetadata(str(video_id), frame_idx, is_empty_frame=False))], + subset=subset, ) proposals[f"{video_id},{frame_idx:04d}"] = [0.0, 0.0, 1.0, 1.0] else: @@ -96,6 +108,7 @@ def generate_action_det_otx_dataset(video_len: int, frame_len: int, labels: List kind=AnnotationSceneKind.ANNOTATION, ), metadata=[MetadataItemEntity(data=VideoMetadata(str(video_id), frame_idx, is_empty_frame=True))], + subset=subset, ) items.append(item) dataset = DatasetEntity(items=items) @@ -117,3 +130,14 @@ def return_args(*args, **kwargs): def return_inputs(inputs): """This function returns its input.""" return inputs + + +def init_environment(params, model_template, label_schema): + """Initialize environment.""" + environment = TaskEnvironment( + model=None, + hyper_parameters=params, + label_schema=label_schema, + model_template=model_template, + ) + return environment diff --git a/tests/unit/algorithms/classification/tasks/test_classification_nncf.py b/tests/unit/algorithms/classification/tasks/test_classification_nncf.py index 84a8e196f5a..2542c8ccbe2 100644 --- a/tests/unit/algorithms/classification/tasks/test_classification_nncf.py +++ b/tests/unit/algorithms/classification/tasks/test_classification_nncf.py @@ -8,7 +8,6 @@ from otx.algorithms.classification.adapters.mmcls.nncf.task import ( ClassificationNNCFTask, ) -from otx.algorithms.common.tasks import BaseTask from otx.api.configuration.configurable_parameters import ConfigurableParameters from otx.api.configuration.helper import create from otx.api.entities.datasets import DatasetEntity @@ -61,7 +60,6 @@ def test_optimize(self, mocker): mock_lcurve_val.x = [0, 1] mock_lcurve_val.y = [0.1, 0.2] # patch training process - mocker.patch.object(BaseTask, "_run_task", return_value={"final_ckpt": ""}) self.cls_nncf_task._learning_curves = {"val/accuracy_top-1": mock_lcurve_val} mocker.patch.object(ClassificationNNCFTask, "save_model") mocker.patch.object(ClassificationNNCFTask, "_train_model") diff --git a/tests/unit/algorithms/common/adapters/mmcv/hooks/test_workflow_hooks.py b/tests/unit/algorithms/common/adapters/mmcv/hooks/test_workflow_hooks.py deleted file mode 100644 index 1fe3709436c..00000000000 --- a/tests/unit/algorithms/common/adapters/mmcv/hooks/test_workflow_hooks.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Unit test for otx.algorithms.common.adapters.mmcv.hooks.workflow_hooks.""" -# Copyright (C) 2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -from otx.algorithms.common.adapters.mmcv.hooks.workflow_hook import ( - AfterStageWFHook, - SampleLoggingHook, - WFProfileHook, - WorkflowHook, - build_workflow_hook, -) -from tests.test_suite.e2e_test_system import e2e_pytest_unit - - -def test_build_workflow_hook() -> None: - try: - build_workflow_hook() - except Exception as e: - print(e) - pass - - -class TestWorkflowHook: - @e2e_pytest_unit - def test_temp(self) -> None: - try: - hook = WorkflowHook() - assert hook is None - except Exception as e: - print(e) - pass - - -class TestSampleLoggingHook: - @e2e_pytest_unit - def test_temp(self) -> None: - try: - hook = SampleLoggingHook() - assert hook is None - except Exception as e: - print(e) - pass - - -class TestWFProfileHook: - @e2e_pytest_unit - def test_temp(self) -> None: - try: - hook = WFProfileHook() - assert hook is None - except Exception as e: - print(e) - pass - - -class TestAfterStageWFHook: - @e2e_pytest_unit - def test_temp(self) -> None: - try: - hook = AfterStageWFHook() - assert hook is None - except Exception as e: - print(e) - pass diff --git a/tests/unit/algorithms/common/adapters/mmcv/tasks/test_builder.py b/tests/unit/algorithms/common/adapters/mmcv/tasks/test_builder.py deleted file mode 100644 index 9ebddb94c01..00000000000 --- a/tests/unit/algorithms/common/adapters/mmcv/tasks/test_builder.py +++ /dev/null @@ -1,37 +0,0 @@ -import mmcv - -from otx.algorithms.common.adapters.mmcv.tasks.builder import build -from tests.test_suite.e2e_test_system import e2e_pytest_unit - - -@e2e_pytest_unit -def test_build_with_stages(mocker): - cfg = mmcv.ConfigDict( - stages=[mocker.MagicMock()], - type=mocker.MagicMock(), - workflow_hooks=[mocker.MagicMock()], - ) - mocker.patch("otx.algorithms.common.adapters.mmcv.tasks.builder.build_workflow_hook") - mock_build_from_cfg = mocker.patch("otx.algorithms.common.adapters.mmcv.tasks.builder.build_from_cfg") - mock_workflow = mocker.patch("otx.algorithms.common.adapters.mmcv.tasks.builder.Workflow") - mocker.patch("otx.algorithms.common.adapters.mmcv.tasks.builder.config_logger") - mocker.patch("os.makedirs") - mocker.patch("os.unlink") - mocker.patch("os.symlink") - - build(cfg) - - mock_build_from_cfg.assert_called() - mock_workflow.assert_called_once() - - -@e2e_pytest_unit -def test_build_without_stages(mocker): - cfg = mmcv.ConfigDict() - - mocker.patch("otx.algorithms.common.adapters.mmcv.tasks.builder.get_available_types", return_value="MockStage") - mock_build_from_cfg = mocker.patch("otx.algorithms.common.adapters.mmcv.tasks.builder.build_from_cfg") - - build(cfg, None, "MockStage") - - mock_build_from_cfg.assert_called_once() diff --git a/tests/unit/algorithms/common/adapters/mmcv/tasks/test_export_mixin.py b/tests/unit/algorithms/common/adapters/mmcv/tasks/test_export_mixin.py deleted file mode 100644 index 3498038d260..00000000000 --- a/tests/unit/algorithms/common/adapters/mmcv/tasks/test_export_mixin.py +++ /dev/null @@ -1,73 +0,0 @@ -import mmcv -import pytest - -from otx.algorithms.common.adapters.mmcv.tasks.exporter_mixin import ExporterMixin -from tests.test_suite.e2e_test_system import e2e_pytest_unit - - -class TestExporterMixin: - @pytest.fixture(autouse=True) - def setup(self, mocker): - def mock_init_logger(): - pass - - def mock_configure(model_cfg, model_ckpt, data_cfg, training=False, **kwargs): - return mmcv.ConfigDict() - - self.exporter = ExporterMixin() - self.exporter._init_logger = mock_init_logger - self.exporter.configure = mock_configure - self.exporter.mode = ["mock_mode", "train"] - fake_config = mmcv.ConfigDict(work_dir="/path/work_dir", data=dict(test=dict(dataset=mocker.MagicMock()))) - mocker.patch.object(self.exporter, "configure", return_value=fake_config) - mocker.patch("os.listdir") - - @e2e_pytest_unit - def test_run_with_error_raise(self): - return_value = self.exporter.run({}, "", {}, mode="mock_mode") - - assert "outputs" in return_value - assert return_value["outputs"] is None - assert "msg" in return_value - - @e2e_pytest_unit - def test_run_without_deploy_cfg(self, mocker): - def mock_naive_export(output_dir, model_builder, precision, cfg, model_name="model"): - pass - - self.exporter.naive_export = mock_naive_export - return_value = self.exporter.run({}, "", {}, mode="mock_mode", model_builder=mocker.MagicMock()) - - assert "outputs" in return_value - assert return_value["outputs"]["bin"] == "/path/work_dir/model.bin" - assert return_value["outputs"]["xml"] == "/path/work_dir/model.xml" - assert "msg" in return_value - assert return_value["msg"] == "" - - @e2e_pytest_unit - def test_run_with_deploy_cfg(self, mocker): - def mock_mmdeploy_export(output_dir, model_builder, precision, cfg, deploy_cfg, model_name="model"): - pass - - self.exporter.mmdeploy_export = mock_mmdeploy_export - return_value = self.exporter.run( - {}, "", {}, mode="mock_mode", model_builder=mocker.MagicMock(), deploy_cfg=mmcv.ConfigDict(deploy=True) - ) - - assert "outputs" in return_value - assert return_value["outputs"]["bin"] == "/path/work_dir/model.bin" - assert return_value["outputs"]["xml"] == "/path/work_dir/model.xml" - assert "msg" in return_value - assert return_value["msg"] == "" - - @e2e_pytest_unit - def test_mmdeploy_export(self, mocker): - from otx.algorithms.common.adapters.mmdeploy.apis import MMdeployExporter - - mock_export_openvino = mocker.patch.object(MMdeployExporter, "export2openvino") - - ExporterMixin.mmdeploy_export( - "", None, "FP16", dict(), mmcv.ConfigDict(backend_config=dict(mo_options=dict(flags=[]))) - ) - - mock_export_openvino.assert_called_once() diff --git a/tests/unit/algorithms/common/adapters/mmcv/tasks/test_stage.py b/tests/unit/algorithms/common/adapters/mmcv/tasks/test_stage.py deleted file mode 100644 index 4bab8b87057..00000000000 --- a/tests/unit/algorithms/common/adapters/mmcv/tasks/test_stage.py +++ /dev/null @@ -1,233 +0,0 @@ -import os - -import mmcv -import pytest - -from otx.algorithms.common.adapters.mmcv.tasks.stage import Stage, get_available_types -from tests.test_suite.e2e_test_system import e2e_pytest_unit - - -@e2e_pytest_unit -def test_get_available_types(): - return_value = get_available_types() - assert isinstance(return_value, list) - - -class TestStage: - @e2e_pytest_unit - def test_init(self, mocker): - fake_cfg = { - "work_dir": "test_init", - "total_epochs": 5, - "runner": {"max_epochs": 10}, - "checkpoint_config": {"interval": 20}, - "seed": 0, - } - fake_common_cfg = {"output_path": "/path/output"} - mocker.patch.object(mmcv, "mkdir_or_exist") - stage = Stage("mpa_test", "", fake_cfg, fake_common_cfg, 0, fake_kwargs=None) - - assert stage.cfg.get("runner", False) - assert stage.cfg.runner.get("max_epochs", 5) - assert stage.cfg.get("checkpoint_config", False) - assert stage.cfg.checkpoint_config.get("interval", 5) - - work_dir = str(stage.cfg.get("work_dir", "")) - - assert work_dir.find("test_init") - assert work_dir.find("mpa_test") - - @e2e_pytest_unit - def test_init_with_distributed_enabled(self, mocker): - fake_cfg = {"work_dir": "test_init_distributed"} - fake_common_cfg = {"output_path": "/path/output"} - mocker.patch.object(mmcv, "mkdir_or_exist") - mocker.patch("torch.distributed.is_initialized", returned_value=True) - os.environ["LOCAL_RANK"] = "2" - stage = Stage("mpa_test", "", fake_cfg, fake_common_cfg, 0) - - assert stage.distributed is True - - @e2e_pytest_unit - def test_configure_data(self): - _base_pipeline = [dict(type="LoadImageFromFile"), dict(type="MultiScaleFlipAug")] - _transform = [ - dict(type="Resize", keep_ratio=False), - dict(type="ImageToTensor", keys=["img"]), - dict(type="Collect", keys=["img"]), - ] - data_cfg = mmcv.ConfigDict( - data=dict( - train=dict(dataset=dict(pipeline=_base_pipeline)), - val=dict(pipeline=_base_pipeline), - pipeline_options=dict( - MultiScaleFlipAug=dict(img_scale=(224, 224), transforms=_transform), - ), - ) - ) - fake_cfg = {"work_dir": "test_init_distributed"} - fake_common_cfg = {"output_path": "/tmp/output"} - stage = Stage("mpa_test", "", fake_cfg, fake_common_cfg, 0) - stage.configure_data(data_cfg, True) - - assert data_cfg.data.train.dataset.pipeline[1].img_scale == (224, 224) - assert data_cfg.data.train.dataset.pipeline[1].transforms == _transform - assert data_cfg.data.val.pipeline[1].img_scale == (224, 224) - assert data_cfg.data.val.pipeline[1].transforms == _transform - - @e2e_pytest_unit - def test_configure_ckpt(self, mocker): - cfg = mmcv.ConfigDict(load_from=None) - mocker.patch.object(mmcv, "mkdir_or_exist") - stage = Stage("mpa_test", "", {}, {}, 0) - - mocker.patch.object(stage, "get_model_ckpt", return_value="/path/to/load/ckpt") - stage.configure_ckpt(cfg, "/foo/bar") - - assert cfg.load_from == "/path/to/load/ckpt" - - @e2e_pytest_unit - def test_configure_hook(self): - cfg = mmcv.ConfigDict( - custom_hook_options=dict(MockHook=dict(opt1="MockOpt1", opt2="MockOpt2")), - custom_hooks=[dict(type="MockHook")], - ) - Stage.configure_hook(cfg) - - assert cfg.custom_hooks[0].opt1 == "MockOpt1" - assert cfg.custom_hooks[0].opt2 == "MockOpt2" - - @e2e_pytest_unit - def test_configure_samples_per_gpu(self, mocker): - cfg = mmcv.ConfigDict(data=dict(train_dataloader=dict(samples_per_gpu=2))) - mock_otx_dataset = mocker.MagicMock() - mock_otx_dataset.__len__.return_value = 1 - - mocker.patch( - "otx.algorithms.common.adapters.mmcv.tasks.stage.get_data_cfg", - return_value=mmcv.ConfigDict(otx_dataset=mock_otx_dataset), - ) - Stage.configure_samples_per_gpu(cfg, "train", False) - - assert "train_dataloader" in cfg.data - assert cfg.data["train_dataloader"]["samples_per_gpu"] == 1 - - @e2e_pytest_unit - def test_configure_compat_cfg(self): - cfg = mmcv.ConfigDict( - data=dict( - train=None, - val=None, - test=None, - unlabeled=None, - ) - ) - with pytest.raises(AttributeError): - Stage.configure_compat_cfg(cfg) - - @e2e_pytest_unit - def test_configure_fp16_optimizer(self): - cfg = mmcv.ConfigDict(fp16=dict(loss_scale=512.0), optimizer_config=dict(type="OptimizerHook")) - - Stage.configure_fp16_optimizer(cfg) - assert cfg.optimizer_config.type == "Fp16OptimizerHook" - - @e2e_pytest_unit - def test_configure_unlabeled_dataloader(self, mocker): - cfg = mmcv.ConfigDict( - data=dict( - unlabeled=dict(), - ), - model_task="classification", - ) - - mocker.patch("importlib.import_module") - mock_build_ul_dataset = mocker.patch("otx.algorithms.common.adapters.mmcv.tasks.stage.build_dataset") - mock_build_ul_dataloader = mocker.patch("otx.algorithms.common.adapters.mmcv.tasks.stage.build_dataloader") - - Stage.configure_unlabeled_dataloader(cfg) - - mock_build_ul_dataset.assert_called_once() - mock_build_ul_dataloader.assert_called_once() - assert cfg.custom_hooks[0].type == "ComposedDataLoadersHook" - - @e2e_pytest_unit - def test_get_model_meta(self, mocker): - cfg = dict(load_from="/foo/bar") - from mmcv.runner import CheckpointLoader - - mock_load_ckpt = mocker.patch.object( - CheckpointLoader, "load_checkpoint", return_value={"meta": {"model_meta": None}} - ) - - returned_value = Stage.get_model_meta(cfg) - - mock_load_ckpt.assert_called_once() - assert returned_value == {"model_meta": None} - - @e2e_pytest_unit - def test_get_data_cfg(self, mocker): - cfg = mmcv.ConfigDict(data=dict(train=dict(dataset=dict(dataset="config")))) - returned_value = Stage.get_data_cfg(cfg, "train") - - assert returned_value == "config" - - @e2e_pytest_unit - def test_get_data_classes(self, mocker): - cfg = mmcv.ConfigDict() - mocker.patch.object(Stage, "get_data_cfg", return_value={"data_classes": ["foo"]}) - - returned_value = Stage.get_data_classes(cfg) - - assert returned_value == ["foo"] - - @e2e_pytest_unit - def test_get_model_classes(self, mocker): - cfg = mmcv.ConfigDict(model=dict()) - mocker.patch.object(Stage, "get_model_meta", return_value={"CLASSES": ["foo", "bar"]}) - mocker.patch.object(Stage, "read_label_schema") - returned_value = Stage.get_model_classes(cfg) - - assert returned_value == ["foo", "bar"] - - @e2e_pytest_unit - def test_get_model_ckpt(self, mocker): - from mmcv.runner import CheckpointLoader - - mock_load_ckpt = mocker.patch.object( - CheckpointLoader, "load_checkpoint", return_value=dict(model=mocker.MagicMock()) - ) - mock_save = mocker.patch("torch.save") - - Stage.get_model_ckpt("/ckpt/path/weights.pth") - - mock_load_ckpt.assert_called_once() - mock_save.assert_called_once() - - @e2e_pytest_unit - def test_read_label_schema(self, mocker): - mocker.patch("os.path.exists", return_value=True) - mocker.patch("builtins.open") - mocker.patch("json.load", return_value=dict(all_labels=dict(label=dict(name="foo")))) - returned_value = Stage.read_label_schema("/ckpt/path/weights.pth") - - assert returned_value == ["foo"] - - @e2e_pytest_unit - def test_set_inference_progress_callback(self, mocker): - mock_model = mocker.MagicMock() - mock_model.register_forward_pre_hook() - mock_model.register_forward_hook() - cfg = dict(custom_hooks=[dict(name="OTXProgressHook", time_monitor=mocker.MagicMock())]) - - Stage.set_inference_progress_callback(cfg, mock_model) - - mock_model.register_forward_pre_hook.assert_called_once() - mock_model.register_forward_hook.assert_called_once() - - @e2e_pytest_unit - def test_build_model(self, mocker): - mock_model_builder = mocker.patch.object(Stage, "MODEL_BUILDER") - Stage.build_model(cfg=mmcv.Config()) - - mock_model_builder.assert_called_once() diff --git a/tests/unit/algorithms/common/adapters/mmcv/tasks/test_workflow.py b/tests/unit/algorithms/common/adapters/mmcv/tasks/test_workflow.py deleted file mode 100644 index 2d61477f7c5..00000000000 --- a/tests/unit/algorithms/common/adapters/mmcv/tasks/test_workflow.py +++ /dev/null @@ -1,38 +0,0 @@ -import mmcv - -from otx.algorithms.common.adapters.mmcv.tasks.stage import Stage -from otx.algorithms.common.adapters.mmcv.tasks.workflow import Workflow -from tests.test_suite.e2e_test_system import e2e_pytest_unit - - -class MockStage(Stage): - def run(*args, **kwargs): - pass - - -class TestWorkflow: - @e2e_pytest_unit - def test_run(self, mocker): - fake_cfg = { - "work_dir": "/path/workdir", - "seed": 0, - } - fake_common_cfg = {"output_path": "/path/output"} - mocker.patch.object(mmcv, "mkdir_or_exist") - stage = MockStage( - "MockStage", - "", - fake_cfg, - fake_common_cfg, - 0, - input=dict( - arg_name1=dict(stage_name="MockStage", output_key=""), - arg_name2=dict(stage_name="MockStage", output_key=""), - ), - ) - workflow = Workflow([stage]) - - mock_stage_run = mocker.patch.object(stage, "run") - workflow.run() - - mock_stage_run.assert_called()