[MPA/OTX] Apply changes in develop to otx branch

mpa/cls/explainer.py

@@ -75,7 +75,7 @@ def _explain(self, cfg):
 
         # InferenceProgressCallback (Time Monitor enable into Infer task)
         ClsStage.set_inference_progress_callback(model, cfg)
-        with self.explainer_hook(model.module.backbone) as forward_explainer_hook:
+        with self.explainer_hook(model.module) as forward_explainer_hook:
             # do inference and record intermediate fmap
             for data in explain_data_loader:
                 with torch.no_grad():

mpa/cls/exporter.py

@@ -70,7 +70,7 @@ def run(self, model_cfg, model_ckpt, data_cfg, **kwargs):
 
         data = self.get_fake_input(cfg)
         fake_img = data['img'].unsqueeze(0)
-        
+
         precision = kwargs.pop('precision', 'FP32')
         logger.info(f'Model will be exported with precision {precision}')
 

mpa/cls/inferrer.py

@@ -16,7 +16,7 @@
 
 from mpa.registry import STAGES
 from mpa.cls.stage import ClsStage
-from mpa.modules.hooks.recording_forward_hooks import ActivationMapHook, FeatureVectorHook
+from mpa.modules.hooks.recording_forward_hooks import FeatureVectorHook, ReciproCAMHook
 from mpa.modules.utils.task_adapt import prob_extractor
 from mpa.utils.logger import get_logger
 logger = get_logger()
@@ -99,9 +99,8 @@ def _infer(self, cfg, dump_features=False, dump_saliency_map=False):
                 data_info['soft_label'] = {task: value[i] for task, value in old_prob.items()}
             outputs = data_infos
         else:
-            with FeatureVectorHook(model.module.backbone) if dump_features else nullcontext() as feature_vector_hook:
-                with ActivationMapHook(model.module.backbone) if dump_saliency_map else nullcontext() \
-                    as forward_explainer_hook:
+            with FeatureVectorHook(model.module) if dump_features else nullcontext() as feature_vector_hook:
+                with ReciproCAMHook(model.module) if dump_saliency_map else nullcontext() as forward_explainer_hook:
                     for data in data_loader:
                         with torch.no_grad():
                             result = model(return_loss=False, **data)

mpa/cls/stage.py

@@ -97,10 +97,6 @@ def is_mmov_model(k, v):
             if cfg.model.get('multilabel', False) or cfg.model.get('hierarchical', False):
                 cfg.model.head.pop('topk', None)
 
-        # Other hyper-parameters
-        if cfg.get('hyperparams', False):
-            self.configure_hyperparams(cfg, training, **kwargs)
-
         return cfg
 
     @staticmethod
@@ -167,7 +163,7 @@ def configure_task(cfg, training, model_meta=None, **kwargs):
 
         model_tasks, dst_classes = None, None
         model_classes, data_classes = [], []
-        train_data_cfg = Stage.get_train_data_cfg(cfg)
+        train_data_cfg = Stage.get_data_cfg(cfg, "train")
         if isinstance(train_data_cfg, list):
             train_data_cfg = train_data_cfg[0]
 
@@ -288,18 +284,6 @@ def configure_task(cfg, training, model_meta=None, **kwargs):
                 cfg.model.head.num_old_classes = len(old_classes)
         return model_tasks, dst_classes
 
-    @staticmethod
-    def configure_hyperparams(cfg, training, **kwargs):
-        hyperparams = kwargs.get('hyperparams', None)
-        if hyperparams is not None:
-            bs = hyperparams.get('bs', None)
-            if bs is not None:
-                cfg.data.samples_per_gpu = bs
-
-            lr = hyperparams.get('lr', None)
-            if lr is not None:
-                cfg.optimizer.lr = lr
-
 
 def refine_tasks(train_cfg, meta, adapt_type):
     new_tasks = train_cfg['tasks']

mpa/cls/trainer.py

@@ -150,9 +150,13 @@ def train_worker(gpu, dataset, cfg, distributed, validate, timestamp, meta):
 
         # prepare data loaders
         dataset = dataset if isinstance(dataset, (list, tuple)) else [dataset]
-        train_data_cfg = Stage.get_train_data_cfg(cfg)
-        drop_last = train_data_cfg.drop_last if train_data_cfg.get('drop_last', False) else False
-
+        train_data_cfg = Stage.get_data_cfg(cfg, "train")
+        ote_dataset = train_data_cfg.get('ote_dataset', None)
+        drop_last = False
+        dataset_len = len(ote_dataset) if ote_dataset else 0
+        # if task == h-label & dataset size is bigger than batch size
+        if train_data_cfg.get('hierarchical_info', None) and dataset_len > cfg.data.get('samples_per_gpu', 2):
+            drop_last = True
         # updated to adapt list of dataset for the 'train'
         data_loaders = []
         sub_loaders = []

mpa/det/exporter.py

@@ -34,6 +34,7 @@ def run(self, model_cfg, model_ckpt, data_cfg, **kwargs):
 
         output_path = os.path.join(cfg.work_dir, 'export')
         os.makedirs(output_path, exist_ok=True)
+
         model = build_detector(cfg.model)
         if model_ckpt:
             load_checkpoint(model=model, filename=model_ckpt, map_location='cpu')

mpa/det/inferrer.py

@@ -1,20 +1,24 @@
 # Copyright (C) 2022 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 #
-from typing import List, Tuple
+from contextlib import nullcontext
 
 import torch
 from mmcv.parallel import MMDataParallel, is_module_wrapper
 from mmcv.runner import load_checkpoint
 
-from mmdet.datasets import build_dataloader, build_dataset, replace_ImageToTensor
+from mmdet.datasets import build_dataloader, build_dataset, replace_ImageToTensor, ImageTilingDataset
 from mmdet.models import build_detector
+from mmdet.models.detectors import TwoStageDetector
 from mmdet.parallel import MMDataCPU
-from mmdet.utils.deployment import get_saliency_map, get_feature_vector
+from mmdet.utils.deployment import get_feature_vector
+from mmdet.apis import single_gpu_test
 
 from mpa.registry import STAGES
 from mpa.utils.logger import get_logger
 from mpa.det.incremental import IncrDetectionStage
+from mpa.modules.hooks.recording_forward_hooks import ActivationMapHook, DetSaliencyMapHook
+
 
 logger = get_logger()
 
@@ -33,9 +37,9 @@ def run(self, model_cfg, model_ckpt, data_cfg, **kwargs):
         """
         self._init_logger()
         mode = kwargs.get('mode', 'train')
-        eval = kwargs.get('eval', False)
-        dump_features = kwargs.get('dump_features', False)
-        dump_saliency_map = kwargs.get('dump_saliency_map', False)
+        eval = kwargs.pop('eval', False)
+        dump_features = kwargs.pop('dump_features', False)
+        dump_saliency_map = kwargs.pop('dump_saliency_map', False)
         if mode not in self.mode:
             return {}
 
@@ -82,7 +86,7 @@ def infer(self, cfg, eval=False, dump_features=False, dump_saliency_map=False):
             input_source = cfg.get('input_source')
             logger.info(f'Inferring on input source: data.{input_source}')
             if input_source == 'train':
-                src_data_cfg = self.get_train_data_cfg(cfg)
+                src_data_cfg = self.get_data_cfg(cfg, input_source)
             else:
                 src_data_cfg = cfg.data[input_source]
             data_cfg.test_mode = src_data_cfg.get('test_mode', False)
@@ -146,7 +150,6 @@ def infer(self, cfg, eval=False, dump_features=False, dump_saliency_map=False):
 
         eval_predictions = []
         feature_vectors = []
-        saliency_maps = []
 
         def dump_features_hook(mod, inp, out):
             with torch.no_grad():
@@ -157,30 +160,24 @@ def dump_features_hook(mod, inp, out):
         def dummy_dump_features_hook(mod, inp, out):
             feature_vectors.append(None)
 
-        def dump_saliency_hook(model: torch.nn.Module, input: Tuple, out: List[torch.Tensor]):
-            """ Dump the last feature map to `saliency_maps` cache
-
-            Args:
-                model (torch.nn.Module): PyTorch model
-                input (Tuple): input
-                out (List[torch.Tensor]): a list of feature maps
-            """
-            with torch.no_grad():
-                saliency_map = get_saliency_map(out[-1])
-            saliency_maps.append(saliency_map.squeeze(0).detach().cpu().numpy())
+        feature_vector_hook = dump_features_hook if dump_features else dummy_dump_features_hook
 
-        def dummy_dump_saliency_hook(model, input, out):
-            saliency_maps.append(None)
+        # Use a single gpu for testing. Set in both mm_val_dataloader and eval_model
+        if is_module_wrapper(model):
+            model = model.module
 
-        feature_vector_hook = dump_features_hook if dump_features else dummy_dump_features_hook
-        saliency_map_hook = dump_saliency_hook if dump_saliency_map else dummy_dump_saliency_hook
+        # Class-wise Saliency map for Single-Stage Detector, otherwise use class-ignore saliency map.
+        if not dump_saliency_map:
+            saliency_hook = nullcontext()
+        elif isinstance(model, TwoStageDetector):
+            saliency_hook = ActivationMapHook(eval_model.module.backbone)
+        else:
+            saliency_hook = DetSaliencyMapHook(eval_model.module)
 
-        with eval_model.module.backbone.register_forward_hook(feature_vector_hook):
-            with eval_model.module.backbone.register_forward_hook(saliency_map_hook):
-                for data in data_loader:
-                    with torch.no_grad():
-                        result = eval_model(return_loss=False, rescale=True, **data)
-                    eval_predictions.extend(result)
+        with eval_model.module.register_forward_hook(feature_vector_hook):
+            with saliency_hook:
+                eval_predictions = single_gpu_test(eval_model, data_loader)
+                saliency_maps = saliency_hook.records if dump_saliency_map else [None] * len(self.dataset)
 
         for key in [
                 'interval', 'tmpdir', 'start', 'gpu_collect', 'save_best',
@@ -190,7 +187,24 @@ def dummy_dump_saliency_hook(model, input, out):
 
         metric = None
         if eval:
-            metric = dataset.evaluate(eval_predictions, **cfg.evaluation)[cfg.evaluation.metric]
+            metric = dataset.evaluate(eval_predictions, **cfg.evaluation)
+            metric = metric['mAP'] if isinstance(cfg.evaluation.metric, list) else metric[cfg.evaluation.metric]
+
+        # Check and unwrap ImageTilingDataset object from TaskAdaptEvalDataset
+        while hasattr(dataset, 'dataset') and not isinstance(dataset, ImageTilingDataset):
+            dataset = dataset.dataset
+
+        if isinstance(dataset, ImageTilingDataset):
+            feature_vectors = [feature_vectors[i] for i in range(dataset.num_samples)]
+            saliency_maps = [saliency_maps[i] for i in range(dataset.num_samples)]
+            if not dataset.merged_results:
+                eval_predictions = dataset.merge(eval_predictions)
+            else:
+                eval_predictions = dataset.merged_results
+
+        assert len(eval_predictions) == len(feature_vectors) == len(saliency_maps), \
+               "Number of elements should be the same, however, number of outputs are " \
+               f"{len(eval_predictions)}, {len(feature_vectors)}, and {len(saliency_maps)}"
 
         outputs = dict(
             classes=target_classes,

mpa/det/trainer.py

@@ -70,6 +70,13 @@ def run(self, model_cfg, model_ckpt, data_cfg, **kwargs):
 
         # Data
         datasets = [build_dataset(cfg.data.train)]
+        cfg.data.val.samples_per_gpu = cfg.data.get('samples_per_gpu', 1)
+
+        # FIXME: scale_factors is fixed at 1 even batch_size > 1 in simple_test_mask
+        # Need to investigate, possibly due to OpenVINO
+        if 'roi_head' in model_cfg.model:
+            if 'mask_head' in model_cfg.model.roi_head:
+                cfg.data.val.samples_per_gpu = 1
 
         if hasattr(cfg, 'hparams'):
             if cfg.hparams.get('adaptive_anchor', False):

mpa/modules/hooks/checkpoint_hook.py

@@ -51,17 +51,17 @@ def __init__(self,
     def after_train_epoch(self, runner):
         if not self.by_epoch or not self.every_n_epochs(runner, self.interval):
             return
-        if hasattr(runner, 'save_ckpt'):
-            if runner.save_ckpt:
-                if runner.save_ema_model:
-                    backup_model = runner.model
-                    runner.model = runner.ema_model
-                runner.logger.info(f'Saving checkpoint at {runner.epoch + 1} epochs')
-                if self.sync_buffer:
-                    allreduce_params(runner.model.buffers())
-                self._save_checkpoint(runner)
-                if runner.save_ema_model:
-                    runner.model = backup_model
+        if hasattr(runner, 'save_ckpt') and runner.save_ckpt:
+            if hasattr(runner, 'save_ema_model') and runner.save_ema_model:
+                backup_model = runner.model
+                runner.model = runner.ema_model
+            runner.logger.info(f'Saving checkpoint at {runner.epoch + 1} epochs')
+            if self.sync_buffer:
+                allreduce_params(runner.model.buffers())
+            self._save_checkpoint(runner)
+            if hasattr(runner, 'save_ema_model') and runner.save_ema_model:
+                runner.model = backup_model
+                runner.save_ema_model = False
             runner.save_ckpt = False
 
     @master_only