From 0351e79f7315c22798f35037f9bde3851dd56cba Mon Sep 17 00:00:00 2001
From: dreamerlin <528557675@qq.com>
Date: Mon, 21 Dec 2020 12:15:53 +0800
Subject: [PATCH 01/17] add EvalHook

---
 mmcv/runner/__init__.py             |   5 +-
 mmcv/runner/hooks/__init__.py       |   4 +-
 mmcv/runner/hooks/eval.py           | 298 ++++++++++++++++++++++++
 tests/test_runner/test_eval_hook.py | 340 ++++++++++++++++++++++++++++
 4 files changed, 644 insertions(+), 3 deletions(-)
 create mode 100644 mmcv/runner/hooks/eval.py
 create mode 100644 tests/test_runner/test_eval_hook.py

diff --git a/mmcv/runner/__init__.py b/mmcv/runner/__init__.py
index df5680ff0b..a8ee907aeb 100644
--- a/mmcv/runner/__init__.py
+++ b/mmcv/runner/__init__.py
@@ -13,7 +13,7 @@
                     EMAHook, Fp16OptimizerHook, Hook, IterTimerHook,
                     LoggerHook, LrUpdaterHook, MlflowLoggerHook, OptimizerHook,
                     PaviLoggerHook, SyncBuffersHook, TensorboardLoggerHook,
-                    TextLoggerHook, WandbLoggerHook)
+                    TextLoggerHook, WandbLoggerHook, EvalHook, DistEvalHook)
 from .iter_based_runner import IterBasedRunner, IterLoader
 from .log_buffer import LogBuffer
 from .optimizer import (OPTIMIZER_BUILDERS, OPTIMIZERS,
@@ -36,5 +36,6 @@
     'set_random_seed', 'auto_fp16', 'force_fp32', 'wrap_fp16_model',
     'Fp16OptimizerHook', 'SyncBuffersHook', 'EMAHook', 'build_runner',
     'RUNNERS', 'allreduce_grads', 'allreduce_params', 'LossScaler',
-    'CheckpointLoader', 'BaseModule', '_load_checkpoint_with_prefix'
+    'CheckpointLoader', 'BaseModule', '_load_checkpoint_with_prefix',
+    'EvalHook', 'DistEvalHook'
 ]
diff --git a/mmcv/runner/hooks/__init__.py b/mmcv/runner/hooks/__init__.py
index c2d5a95144..e57d58faf2 100644
--- a/mmcv/runner/hooks/__init__.py
+++ b/mmcv/runner/hooks/__init__.py
@@ -2,6 +2,7 @@
 from .checkpoint import CheckpointHook
 from .closure import ClosureHook
 from .ema import EMAHook
+from .eval import EvalHook, DistEvalHook
 from .hook import HOOKS, Hook
 from .iter_timer import IterTimerHook
 from .logger import (LoggerHook, MlflowLoggerHook, PaviLoggerHook,
@@ -18,5 +19,6 @@
     'OptimizerHook', 'Fp16OptimizerHook', 'IterTimerHook',
     'DistSamplerSeedHook', 'EmptyCacheHook', 'LoggerHook', 'MlflowLoggerHook',
     'PaviLoggerHook', 'TextLoggerHook', 'TensorboardLoggerHook',
-    'WandbLoggerHook', 'MomentumUpdaterHook', 'SyncBuffersHook', 'EMAHook'
+    'WandbLoggerHook', 'MomentumUpdaterHook', 'SyncBuffersHook', 'EMAHook',
+    'EvalHook', 'DistEvalHook'
 ]
diff --git a/mmcv/runner/hooks/eval.py b/mmcv/runner/hooks/eval.py
new file mode 100644
index 0000000000..1e6a302010
--- /dev/null
+++ b/mmcv/runner/hooks/eval.py
@@ -0,0 +1,298 @@
+import os.path as osp
+import warnings
+from math import inf
+
+from mmcv.runner import Hook
+from mmcv import symlink
+from mmcv.engine import single_gpu_test, multi_gpu_test
+from torch.utils.data import DataLoader
+
+
+class EvalHook(Hook):
+    """Non-Distributed evaluation hook.
+    Notes:
+        If new arguments are added for EvalHook, tools/test.py,
+        tools/eval_metric.py may be effected.
+    This hook will regularly perform evaluation in a given interval when
+    performing in non-distributed environment.
+    Args:
+        dataloader (DataLoader): A PyTorch dataloader.
+        start (int | None, optional): Evaluation starting epoch. It enables
+            evaluation before the training starts if ``start`` <= the resuming
+            epoch. If None, whether to evaluate is merely decided by
+            ``interval``. Default: None.
+        interval (int): Evaluation interval. Default: 1.
+        by_epoch (bool): Determine perform evaluation by epoch or by iteration.
+            If set to True, it will perform by epoch. Otherwise, by iteration.
+            default: True.
+        save_best (str | None, optional): If a metric is specified, it would
+            measure the best checkpoint during evaluation. The information
+            about best checkpoint would be save in best.json.
+            Options are the evaluation metrics to the test dataset. e.g.,
+             ``top1_acc``, ``top5_acc``, ``mean_class_accuracy``,
+            ``mean_average_precision``, ``mmit_mean_average_precision``
+            for action recognition dataset (RawframeDataset and VideoDataset).
+            ``AR@AN``, ``auc`` for action localization dataset.
+            (ActivityNetDataset). Default: None.
+        rule (str | None, optional): Comparison rule for best score. If set to
+            None, it will infer a reasonable rule. Keys such as 'acc', 'top'
+            .etc will be inferred by 'greater' rule. Keys contain 'loss' will
+            be inferred by 'less' rule. Options are 'greater', 'less', None.
+            Default: None.
+        **eval_kwargs: Evaluation arguments fed into the evaluate function of
+            the dataset.
+    """
+
+    rule_map = {'greater': lambda x, y: x > y, 'less': lambda x, y: x < y}
+    init_value_map = {'greater': -inf, 'less': inf}
+    greater_keys = ['acc', 'top', 'AR@', 'auc', 'precision']
+    less_keys = ['loss']
+
+    def __init__(self,
+                 dataloader,
+                 start=None,
+                 interval=1,
+                 by_epoch=True,
+                 save_best=None,
+                 rule=None,
+                 **eval_kwargs):
+        if not isinstance(dataloader, DataLoader):
+            raise TypeError(f'dataloader must be a pytorch DataLoader, '
+                            f'but got {type(dataloader)}')
+
+        if interval <= 0:
+            raise ValueError(f'interval must be positive, but got {interval}')
+
+        assert isinstance(by_epoch, bool)
+
+        if start is not None and start < 0:
+            warnings.warn(
+                f'The evaluation start epoch {start} is smaller than 0, '
+                f'use 0 instead', UserWarning)
+            start = 0
+        self.dataloader = dataloader
+        self.interval = interval
+        self.start = start
+        self.by_epoch = by_epoch
+
+        assert isinstance(save_best, str) or save_best is None
+        self.save_best = save_best
+        self.eval_kwargs = eval_kwargs
+        self.initial_epoch_flag = True
+
+        if self.save_best is not None:
+            self._init_rule(rule, self.save_best)
+
+    def _init_rule(self, rule, key_indicator):
+        """Initialize rule, key_indicator, comparison_func, and best score.
+        Args:
+            rule (str | None): Comparison rule for best score.
+            key_indicator (str | None): Key indicator to determine the
+                comparison rule.
+        """
+        if rule not in self.rule_map and rule is not None:
+            raise KeyError(f'rule must be greater, less or None, '
+                           f'but got {rule}.')
+
+        if rule is None:
+            if key_indicator != 'auto':
+                if any(key in key_indicator for key in self.greater_keys):
+                    rule = 'greater'
+                elif any(key in key_indicator for key in self.less_keys):
+                    rule = 'less'
+                else:
+                    raise ValueError(f'Cannot infer the rule for key '
+                                     f'{key_indicator}, thus a specific rule '
+                                     f'must be specified.')
+        self.rule = rule
+        self.key_indicator = key_indicator
+        if self.rule is not None:
+            self.compare_func = self.rule_map[self.rule]
+
+    def before_run(self, runner):
+        if self.save_best is not None:
+            if runner.meta is None:
+                warnings.warn('runner.meta is None. Creating a empty one.')
+                runner.meta = dict()
+            runner.meta.setdefault('hook_msgs', dict())
+
+    def before_train_iter(self, runner):
+        """Evaluate the model only at the start of training by iteration."""
+        if self.by_epoch:
+            return
+        if not self.initial_epoch_flag:
+            return
+        if self.start is not None and runner.iter >= self.start:
+            self.after_train_iter(runner)
+        self.initial_epoch_flag = False
+
+    def before_train_epoch(self, runner):
+        """Evaluate the model only at the start of training by epoch."""
+        if not self.by_epoch:
+            return
+        if not self.initial_epoch_flag:
+            return
+        if self.start is not None and runner.epoch >= self.start:
+            self.after_train_epoch(runner)
+        self.initial_epoch_flag = False
+
+    def after_train_iter(self, runner):
+        """Called after every training iter to evaluate the results."""
+        if not self.by_epoch:
+            self._do_evaluate(runner)
+
+    def after_train_epoch(self, runner):
+        """Called after every training epoch to evaluate the results."""
+        if self.by_epoch:
+            self._do_evaluate(runner)
+
+    def _do_evaluate(self, runner):
+        """perform evaluation and save ckpt."""
+        if not self.evaluation_flag(runner):
+            return
+
+        results = single_gpu_test(runner.model, self.dataloader)
+        key_score = self.evaluate(runner, results)
+        if self.save_best:
+            self._save_ckpt(runner, key_score)
+
+    def evaluation_flag(self, runner):
+        """Judge whether to perform_evaluation.
+        Returns:
+            bool: The flag indicating whether to perform evaluation.
+        """
+        if self.by_epoch:
+            current = runner.epoch
+            check_time = self.every_n_epochs
+        else:
+            current = runner.iter
+            check_time = self.every_n_iters
+
+        if self.start is None:
+            if not check_time(runner, self.interval):
+                # No evaluation during the interval.
+                return False
+        elif (current + 1) < self.start:
+            # No evaluation if start is larger than the current time.
+            return False
+        else:
+            # Evaluation only at epochs 3, 5, 7... if start==3 and interval==2
+            if (current + 1 - self.start) % self.interval:
+                return False
+        return True
+
+    def _save_ckpt(self, runner, key_score):
+        if self.by_epoch:
+            current = f'epoch_{runner.epoch + 1}'
+        else:
+            current = f'iter_{runner.epoch + 1}'
+
+        best_score = runner.meta['hook_msgs'].get(
+            'best_score', self.init_value_map[self.rule])
+        if self.compare_func(key_score, best_score):
+            best_score = key_score
+            runner.meta['hook_msgs']['best_score'] = best_score
+            last_ckpt = runner.meta['hook_msgs']['last_ckpt']
+            runner.meta['hook_msgs']['best_ckpt'] = last_ckpt
+            symlink(
+                last_ckpt,
+                osp.join(runner.work_dir, f'best_{self.key_indicator}.pth'))
+            runner.logger.info(f'Now best checkpoint is {current}.pth.'
+                               f'Best {self.key_indicator} is {best_score:0.4f}')
+
+    def evaluate(self, runner, results):
+        """Evaluate the results.
+        Args:
+            runner (:obj:`mmcv.Runner`): The underlined training runner.
+            results (list): Output results.
+        """
+        eval_res = self.dataloader.dataset.evaluate(
+            results, logger=runner.logger, **self.eval_kwargs)
+        for name, val in eval_res.items():
+            runner.log_buffer.output[name] = val
+        runner.log_buffer.ready = True
+        if self.save_best is not None:
+            if self.key_indicator == 'auto':
+                # infer from eval_results
+                self._init_rule(self.rule, list(eval_res.keys())[0])
+            return eval_res[self.key_indicator]
+
+        return None
+
+
+class DistEvalHook(EvalHook):
+    """Distributed evaluation hook.
+    This hook will regularly perform evaluation in a given interval when
+    performing in distributed environment.
+    Args:
+        dataloader (DataLoader): A PyTorch dataloader.
+        start (int | None, optional): Evaluation starting epoch. It enables
+            evaluation before the training starts if ``start`` <= the resuming
+            epoch. If None, whether to evaluate is merely decided by
+            ``interval``. Default: None.
+        interval (int): Evaluation interval. Default: 1.
+        by_epoch (bool): Determine perform evaluation by epoch or by iteration.
+            If set to True, it will perform by epoch. Otherwise, by iteration.
+            default: True.
+        save_best (str | None, optional): If a metric is specified, it would
+            measure the best checkpoint during evaluation. The information
+            about best checkpoint would be save in best.json.
+            Options are the evaluation metrics to the test dataset. e.g.,
+             ``top1_acc``, ``top5_acc``, ``mean_class_accuracy``,
+            ``mean_average_precision``, ``mmit_mean_average_precision``
+            for action recognition dataset (RawframeDataset and VideoDataset).
+            ``AR@AN``, ``auc`` for action localization dataset.
+            (ActivityNetDataset). Default: None.
+        rule (str | None, optional): Comparison rule for best score. If set to
+            None, it will infer a reasonable rule. Keys such as 'acc', 'top'
+            .etc will be inferred by 'greater' rule. Keys contain 'loss' will
+            be inferred by 'less' rule. Options are 'greater', 'less', None.
+            Default: None.
+        tmpdir (str | None): Temporary directory to save the results of all
+            processes. Default: None.
+        gpu_collect (bool): Whether to use gpu or cpu to collect results.
+            Default: False.
+        **eval_kwargs: Evaluation arguments fed into the evaluate function of
+            the dataset.
+    """
+
+    def __init__(self,
+                 dataloader,
+                 start=None,
+                 interval=1,
+                 by_epoch=True,
+                 save_best=None,
+                 rule=None,
+                 tmpdir=None,
+                 gpu_collect=False,
+                 **eval_kwargs):
+        super().__init__(
+            dataloader,
+            start=start,
+            interval=interval,
+            by_epoch=by_epoch,
+            save_best=save_best,
+            rule=rule,
+            **eval_kwargs)
+        self.tmpdir = tmpdir
+        self.gpu_collect = gpu_collect
+
+    def _do_evaluate(self, runner):
+        if not self.evaluation_flag(runner):
+            return
+
+        tmpdir = self.tmpdir
+        if tmpdir is None:
+            tmpdir = osp.join(runner.work_dir, '.eval_hook')
+
+        results = multi_gpu_test(
+            runner.model,
+            self.dataloader,
+            tmpdir=tmpdir,
+            gpu_collect=self.gpu_collect)
+        if runner.rank == 0:
+            print('\n')
+            key_score = self.evaluate(runner, results)
+
+            if self.save_best:
+                self._save_ckpt(runner, key_score)
diff --git a/tests/test_runner/test_eval_hook.py b/tests/test_runner/test_eval_hook.py
new file mode 100644
index 0000000000..6e066af6a3
--- /dev/null
+++ b/tests/test_runner/test_eval_hook.py
@@ -0,0 +1,340 @@
+import os.path as osp
+import tempfile
+import unittest.mock as mock
+from collections import OrderedDict
+from unittest.mock import MagicMock, patch
+
+import pytest
+import torch
+import torch.nn as nn
+from mmcv.runner import EpochBasedRunner, IterBasedRunner, EvalHook, DistEvalHook
+from mmcv.utils import get_logger
+from torch.utils.data import DataLoader, Dataset
+
+
+class ExampleDataset(Dataset):
+
+    def __init__(self):
+        self.index = 0
+        self.eval_result = [1, 4, 3, 7, 2, -3, 4, 6]
+
+    def __getitem__(self, idx):
+        results = dict(x=torch.tensor([1]))
+        return results
+
+    def __len__(self):
+        return 1
+
+    @mock.create_autospec
+    def evaluate(self, results, logger=None):
+        pass
+
+
+class EvalDataset(ExampleDataset):
+
+    def evaluate(self, results, logger=None):
+        acc = self.eval_result[self.index]
+        output = OrderedDict(acc=acc, index=self.index, score=acc)
+        self.index += 1
+        return output
+
+
+class Model(nn.Module):
+
+    def __init__(self):
+        super().__init__()
+        self.linear = nn.Linear(2, 1)
+
+    def forward(self, x, **kwargs):
+        return x
+
+    def train_step(self, data_batch, optimizer, **kwargs):
+        if not isinstance(data_batch, dict):
+            data_batch = dict(x=data_batch)
+        return data_batch
+
+    def val_step(self, x, optimizer, **kwargs):
+        return dict(loss=self(x))
+
+
+def _build_epoch_runner():
+
+    model = Model()
+    tmp_dir = tempfile.mkdtemp()
+
+    runner = EpochBasedRunner(
+        model=model, work_dir=tmp_dir, logger=get_logger('demo'))
+    return runner
+
+
+def _build_iter_runner():
+
+    model = Model()
+    tmp_dir = tempfile.mkdtemp()
+
+    runner = IterBasedRunner(
+        model=model, work_dir=tmp_dir, logger=get_logger('demo'))
+    return runner
+
+
+def test_eval_hook():
+    with pytest.raises(AssertionError):
+        # `save_best` should be a str
+        test_dataset = Model()
+        data_loader = DataLoader(test_dataset)
+        EvalHook(data_loader, save_best=True)
+
+    with pytest.raises(TypeError):
+        # dataloader must be a pytorch DataLoader
+        test_dataset = Model()
+        data_loader = [DataLoader(test_dataset)]
+        EvalHook(data_loader)
+
+    with pytest.raises(ValueError):
+        # key_indicator must be valid when rule_map is None
+        test_dataset = ExampleDataset()
+        data_loader = DataLoader(test_dataset)
+        EvalHook(data_loader, save_best='unsupport')
+
+    with pytest.raises(KeyError):
+        # rule must be in keys of rule_map
+        test_dataset = Model()
+        data_loader = DataLoader(test_dataset)
+        EvalHook(data_loader, save_best='auto', rule='unsupport')
+
+    test_dataset = ExampleDataset()
+    loader = DataLoader(test_dataset)
+    model = Model()
+    data_loader = DataLoader(test_dataset)
+    eval_hook = EvalHook(data_loader, save_best=None)
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+
+        # total_epochs = 1
+        logger = get_logger('test_eval')
+        runner = EpochBasedRunner(model=model, work_dir=tmpdir, logger=logger)
+        runner.register_hook(eval_hook)
+        runner.run([loader], [('train', 1)], 1)
+        test_dataset.evaluate.assert_called_with(
+            test_dataset, [torch.tensor([1])], logger=runner.logger)
+        assert runner.meta is None or 'best_score' not in runner.meta[
+            'hook_msgs']
+        assert runner.meta is None or 'best_ckpt' not in runner.meta[
+            'hook_msgs']
+
+        # when `save_best` is set to 'auto', first metric will be used.
+        loader = DataLoader(EvalDataset())
+        model = Model()
+        data_loader = DataLoader(EvalDataset())
+        eval_hook = EvalHook(data_loader, interval=1, save_best='auto')
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            logger = get_logger('test_eval')
+            runner = EpochBasedRunner(
+                model=model, work_dir=tmpdir, logger=logger)
+            runner.register_checkpoint_hook(dict(interval=1))
+            runner.register_hook(eval_hook)
+            runner.run([loader], [('train', 1)], 8)
+
+            real_path = osp.join(tmpdir, 'epoch_4.pth')
+            link_path = osp.join(tmpdir, 'best_acc.pth')
+
+            assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(
+                real_path)
+            assert osp.exists(link_path)
+            assert runner.meta['hook_msgs']['best_score'] == 7
+
+        # total_epochs = 8, return the best acc and corresponding epoch
+        loader = DataLoader(EvalDataset())
+        model = Model()
+        data_loader = DataLoader(EvalDataset())
+        eval_hook = EvalHook(data_loader, interval=1, save_best='acc')
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            logger = get_logger('test_eval')
+            runner = EpochBasedRunner(
+                model=model, work_dir=tmpdir, logger=logger)
+            runner.register_checkpoint_hook(dict(interval=1))
+            runner.register_hook(eval_hook)
+            runner.run([loader], [('train', 1)], 8)
+
+            real_path = osp.join(tmpdir, 'epoch_4.pth')
+            link_path = osp.join(tmpdir, 'best_acc.pth')
+
+            assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(
+                real_path)
+            assert osp.exists(link_path)
+            assert runner.meta['hook_msgs']['best_score'] == 7
+
+        # total_epochs = 8, return the best score and corresponding epoch
+        data_loader = DataLoader(EvalDataset())
+        eval_hook = EvalHook(
+            data_loader, interval=1, save_best='score', rule='greater')
+        with tempfile.TemporaryDirectory() as tmpdir:
+            logger = get_logger('test_eval')
+            runner = EpochBasedRunner(
+                model=model, work_dir=tmpdir, logger=logger)
+            runner.register_checkpoint_hook(dict(interval=1))
+            runner.register_hook(eval_hook)
+            runner.run([loader], [('train', 1)], 8)
+
+            real_path = osp.join(tmpdir, 'epoch_4.pth')
+            link_path = osp.join(tmpdir, 'best_score.pth')
+
+            assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(
+                real_path)
+            assert osp.exists(link_path)
+            assert runner.meta['hook_msgs']['best_score'] == 7
+
+        # total_epochs = 8, return the best score using less compare func
+        # and indicate corresponding epoch
+        data_loader = DataLoader(EvalDataset())
+        eval_hook = EvalHook(data_loader, save_best='acc', rule='less')
+        with tempfile.TemporaryDirectory() as tmpdir:
+            logger = get_logger('test_eval')
+            runner = EpochBasedRunner(
+                model=model, work_dir=tmpdir, logger=logger)
+            runner.register_checkpoint_hook(dict(interval=1))
+            runner.register_hook(eval_hook)
+            runner.run([loader], [('train', 1)], 8)
+
+            real_path = osp.join(tmpdir, 'epoch_6.pth')
+            link_path = osp.join(tmpdir, 'best_acc.pth')
+
+            assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(
+                real_path)
+            assert osp.exists(link_path)
+            assert runner.meta['hook_msgs']['best_score'] == -3
+
+        # Test the EvalHook when resume happend
+        data_loader = DataLoader(EvalDataset())
+        eval_hook = EvalHook(data_loader, save_best='acc')
+        with tempfile.TemporaryDirectory() as tmpdir:
+            logger = get_logger('test_eval')
+            runner = EpochBasedRunner(
+                model=model, work_dir=tmpdir, logger=logger)
+            runner.register_checkpoint_hook(dict(interval=1))
+            runner.register_hook(eval_hook)
+            runner.run([loader], [('train', 1)], 2)
+
+            real_path = osp.join(tmpdir, 'epoch_2.pth')
+            link_path = osp.join(tmpdir, 'best_acc.pth')
+
+            assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(
+                real_path)
+            assert osp.exists(link_path)
+            assert runner.meta['hook_msgs']['best_score'] == 4
+
+            resume_from = osp.join(tmpdir, 'latest.pth')
+            loader = DataLoader(ExampleDataset())
+            eval_hook = EvalHook(data_loader, save_best='acc')
+            runner = EpochBasedRunner(
+                model=model, work_dir=tmpdir, logger=logger)
+            runner.register_checkpoint_hook(dict(interval=1))
+            runner.register_hook(eval_hook)
+            runner.resume(resume_from)
+            runner.run([loader], [('train', 1)], 8)
+
+            real_path = osp.join(tmpdir, 'epoch_4.pth')
+            link_path = osp.join(tmpdir, 'best_acc.pth')
+
+            assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(
+                real_path)
+            assert osp.exists(link_path)
+            assert runner.meta['hook_msgs']['best_score'] == 7
+
+
+@patch('mmcv.engine.single_gpu_test', MagicMock)
+@patch('mmcv.engine.multi_gpu_test', MagicMock)
+@pytest.mark.parametrize('EvalHookParam', [EvalHook, DistEvalHook])
+@pytest.mark.parametrize('_build_demo_runner,by_epoch',
+                         [(_build_epoch_runner, True),
+                          (_build_iter_runner, False)])
+def test_start_param(EvalHookParam, _build_demo_runner, by_epoch):
+    # create dummy data
+    dataloader = DataLoader(torch.ones((5, 2)))
+
+    # 0.1. dataloader is not a DataLoader object
+    with pytest.raises(TypeError):
+        EvalHookParam(dataloader=MagicMock(), interval=-1)
+
+    # 0.2. negative interval
+    with pytest.raises(ValueError):
+        EvalHookParam(dataloader, interval=-1)
+
+    # 1. start=None, interval=1: perform evaluation after each epoch.
+    runner = _build_demo_runner()
+    evalhook = EvalHookParam(dataloader, interval=1, by_epoch=by_epoch)
+    evalhook.evaluate = MagicMock()
+    runner.register_hook(evalhook)
+    runner.run([dataloader], [('train', 1)], 2)
+    assert evalhook.evaluate.call_count == 2  # after epoch 1 & 2
+
+    # 2. start=1, interval=1: perform evaluation after each epoch.
+    runner = _build_demo_runner()
+    evalhook = EvalHookParam(
+        dataloader, start=1, interval=1, by_epoch=by_epoch)
+    evalhook.evaluate = MagicMock()
+    runner.register_hook(evalhook)
+    runner.run([dataloader], [('train', 1)], 2)
+    assert evalhook.evaluate.call_count == 2  # after epoch 1 & 2
+
+    # 3. start=None, interval=2: perform evaluation after epoch 2, 4, 6, etc
+    runner = _build_demo_runner()
+    evalhook = EvalHookParam(dataloader, interval=2, by_epoch=by_epoch)
+    evalhook.evaluate = MagicMock()
+    runner.register_hook(evalhook)
+    runner.run([dataloader], [('train', 1)], 2)
+    assert evalhook.evaluate.call_count == 1  # after epoch 2
+
+    # 4. start=1, interval=2: perform evaluation after epoch 1, 3, 5, etc
+    runner = _build_demo_runner()
+    evalhook = EvalHookParam(
+        dataloader, start=1, interval=2, by_epoch=by_epoch)
+    evalhook.evaluate = MagicMock()
+    runner.register_hook(evalhook)
+    runner.run([dataloader], [('train', 1)], 3)
+    assert evalhook.evaluate.call_count == 2  # after epoch 1 & 3
+
+    # 5. start=0/negative, interval=1: perform evaluation after each epoch and
+    #    before epoch 1.
+    runner = _build_demo_runner()
+    evalhook = EvalHookParam(dataloader, start=0, by_epoch=by_epoch)
+    evalhook.evaluate = MagicMock()
+    runner.register_hook(evalhook)
+    runner.run([dataloader], [('train', 1)], 2)
+    assert evalhook.evaluate.call_count == 3  # before epoch1 and after e1 & e2
+
+    runner = _build_demo_runner()
+    with pytest.warns(UserWarning):
+        evalhook = EvalHookParam(dataloader, start=-2, by_epoch=by_epoch)
+    evalhook.evaluate = MagicMock()
+    runner.register_hook(evalhook)
+    runner.run([dataloader], [('train', 1)], 2)
+    assert evalhook.evaluate.call_count == 3  # before epoch1 and after e1 & e2
+
+    # 6. resuming from epoch i, start = x (x<=i), interval =1: perform
+    #    evaluation after each epoch and before the first epoch.
+    runner = _build_demo_runner()
+    evalhook = EvalHookParam(dataloader, start=1, by_epoch=by_epoch)
+    evalhook.evaluate = MagicMock()
+    runner.register_hook(evalhook)
+    if by_epoch:
+        runner._epoch = 2
+    else:
+        runner._iter = 2
+    runner.run([dataloader], [('train', 1)], 3)
+    assert evalhook.evaluate.call_count == 2  # before & after epoch 3
+
+    # 7. resuming from epoch i, start = i+1/None, interval =1: perform
+    #    evaluation after each epoch.
+    runner = _build_demo_runner()
+    evalhook = EvalHookParam(dataloader, start=2, by_epoch=by_epoch)
+    evalhook.evaluate = MagicMock()
+    runner.register_hook(evalhook)
+    if by_epoch:
+        runner._epoch = 1
+    else:
+        runner._iter = 1
+    runner.run([dataloader], [('train', 1)], 3)
+    assert evalhook.evaluate.call_count == 2  # after epoch 2 & 3
\ No newline at end of file

From f1e23260f4ac25616448c25cc58c5246cfa6bb8e Mon Sep 17 00:00:00 2001
From: dreamerlin <528557675@qq.com>
Date: Mon, 21 Dec 2020 12:17:49 +0800
Subject: [PATCH 02/17] add EvalHook

---
 docs/readme.md                      |  2 +-
 mmcv/runner/__init__.py             | 11 ++++++-----
 mmcv/runner/hooks/__init__.py       |  2 +-
 mmcv/runner/hooks/eval.py           | 15 ++++++++++-----
 tests/test_runner/test_eval_hook.py |  8 +++++---
 5 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/docs/readme.md b/docs/readme.md
index 32d46ee883..94389aee61 120000
--- a/docs/readme.md
+++ b/docs/readme.md
@@ -1 +1 @@
-../README.md
\ No newline at end of file
+../README.md
diff --git a/mmcv/runner/__init__.py b/mmcv/runner/__init__.py
index a8ee907aeb..df087cf16e 100644
--- a/mmcv/runner/__init__.py
+++ b/mmcv/runner/__init__.py
@@ -9,11 +9,12 @@
                          init_dist, master_only)
 from .epoch_based_runner import EpochBasedRunner, Runner
 from .fp16_utils import LossScaler, auto_fp16, force_fp32, wrap_fp16_model
-from .hooks import (HOOKS, CheckpointHook, ClosureHook, DistSamplerSeedHook,
-                    EMAHook, Fp16OptimizerHook, Hook, IterTimerHook,
-                    LoggerHook, LrUpdaterHook, MlflowLoggerHook, OptimizerHook,
-                    PaviLoggerHook, SyncBuffersHook, TensorboardLoggerHook,
-                    TextLoggerHook, WandbLoggerHook, EvalHook, DistEvalHook)
+from .hooks import (HOOKS, CheckpointHook, ClosureHook, DistEvalHook,
+                    DistSamplerSeedHook, EMAHook, EvalHook, Fp16OptimizerHook,
+                    Hook, IterTimerHook, LoggerHook, LrUpdaterHook,
+                    MlflowLoggerHook, OptimizerHook, PaviLoggerHook,
+                    SyncBuffersHook, TensorboardLoggerHook, TextLoggerHook,
+                    WandbLoggerHook)
 from .iter_based_runner import IterBasedRunner, IterLoader
 from .log_buffer import LogBuffer
 from .optimizer import (OPTIMIZER_BUILDERS, OPTIMIZERS,
diff --git a/mmcv/runner/hooks/__init__.py b/mmcv/runner/hooks/__init__.py
index e57d58faf2..6ece21fcb9 100644
--- a/mmcv/runner/hooks/__init__.py
+++ b/mmcv/runner/hooks/__init__.py
@@ -2,7 +2,7 @@
 from .checkpoint import CheckpointHook
 from .closure import ClosureHook
 from .ema import EMAHook
-from .eval import EvalHook, DistEvalHook
+from .eval import DistEvalHook, EvalHook
 from .hook import HOOKS, Hook
 from .iter_timer import IterTimerHook
 from .logger import (LoggerHook, MlflowLoggerHook, PaviLoggerHook,
diff --git a/mmcv/runner/hooks/eval.py b/mmcv/runner/hooks/eval.py
index 1e6a302010..1d47719ed4 100644
--- a/mmcv/runner/hooks/eval.py
+++ b/mmcv/runner/hooks/eval.py
@@ -2,11 +2,12 @@
 import warnings
 from math import inf
 
-from mmcv.runner import Hook
-from mmcv import symlink
-from mmcv.engine import single_gpu_test, multi_gpu_test
 from torch.utils.data import DataLoader
 
+from mmcv import symlink
+from mmcv.engine import multi_gpu_test, single_gpu_test
+from mmcv.runner import Hook
+
 
 class EvalHook(Hook):
     """Non-Distributed evaluation hook.
@@ -85,6 +86,7 @@ def __init__(self,
 
     def _init_rule(self, rule, key_indicator):
         """Initialize rule, key_indicator, comparison_func, and best score.
+
         Args:
             rule (str | None): Comparison rule for best score.
             key_indicator (str | None): Key indicator to determine the
@@ -158,6 +160,7 @@ def _do_evaluate(self, runner):
 
     def evaluation_flag(self, runner):
         """Judge whether to perform_evaluation.
+
         Returns:
             bool: The flag indicating whether to perform evaluation.
         """
@@ -197,11 +200,13 @@ def _save_ckpt(self, runner, key_score):
             symlink(
                 last_ckpt,
                 osp.join(runner.work_dir, f'best_{self.key_indicator}.pth'))
-            runner.logger.info(f'Now best checkpoint is {current}.pth.'
-                               f'Best {self.key_indicator} is {best_score:0.4f}')
+            runner.logger.info(
+                f'Now best checkpoint is {current}.pth.'
+                f'Best {self.key_indicator} is {best_score:0.4f}')
 
     def evaluate(self, runner, results):
         """Evaluate the results.
+
         Args:
             runner (:obj:`mmcv.Runner`): The underlined training runner.
             results (list): Output results.
diff --git a/tests/test_runner/test_eval_hook.py b/tests/test_runner/test_eval_hook.py
index 6e066af6a3..b359942bb2 100644
--- a/tests/test_runner/test_eval_hook.py
+++ b/tests/test_runner/test_eval_hook.py
@@ -7,10 +7,12 @@
 import pytest
 import torch
 import torch.nn as nn
-from mmcv.runner import EpochBasedRunner, IterBasedRunner, EvalHook, DistEvalHook
-from mmcv.utils import get_logger
 from torch.utils.data import DataLoader, Dataset
 
+from mmcv.runner import (DistEvalHook, EpochBasedRunner, EvalHook,
+                         IterBasedRunner)
+from mmcv.utils import get_logger
+
 
 class ExampleDataset(Dataset):
 
@@ -337,4 +339,4 @@ def test_start_param(EvalHookParam, _build_demo_runner, by_epoch):
     else:
         runner._iter = 1
     runner.run([dataloader], [('train', 1)], 3)
-    assert evalhook.evaluate.call_count == 2  # after epoch 2 & 3
\ No newline at end of file
+    assert evalhook.evaluate.call_count == 2  # after epoch 2 & 3

From a3e6e15627df3ac0a37d9f9adf7f84f4b57abdc1 Mon Sep 17 00:00:00 2001
From: dreamerlin <528557675@qq.com>
Date: Wed, 23 Dec 2020 11:48:04 +0800
Subject: [PATCH 03/17] refactor docstring

---
 mmcv/runner/hooks/eval.py | 30 ++++++++++++++----------------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/mmcv/runner/hooks/eval.py b/mmcv/runner/hooks/eval.py
index 1d47719ed4..114406aee6 100644
--- a/mmcv/runner/hooks/eval.py
+++ b/mmcv/runner/hooks/eval.py
@@ -26,15 +26,14 @@ class EvalHook(Hook):
         by_epoch (bool): Determine perform evaluation by epoch or by iteration.
             If set to True, it will perform by epoch. Otherwise, by iteration.
             default: True.
-        save_best (str | None, optional): If a metric is specified, it would
-            measure the best checkpoint during evaluation. The information
-            about best checkpoint would be save in best.json.
+        save_best (str, optional): If a metric is specified, it would measure
+            the best checkpoint during evaluation. The information about best
+            checkpoint would be save in best.json.
             Options are the evaluation metrics to the test dataset. e.g.,
-             ``top1_acc``, ``top5_acc``, ``mean_class_accuracy``,
-            ``mean_average_precision``, ``mmit_mean_average_precision``
-            for action recognition dataset (RawframeDataset and VideoDataset).
-            ``AR@AN``, ``auc`` for action localization dataset.
-            (ActivityNetDataset). Default: None.
+            ``bbox_mAP``, ``segm_mAP`` for bbox detection and instance
+            segmentation. ``AR@100`` for proposal recall. If ``save_best`` is
+            ``auto``, the first key will be used. The interval of
+            ``CheckpointHook`` should device EvalHook. Default: None.
         rule (str | None, optional): Comparison rule for best score. If set to
             None, it will infer a reasonable rule. Keys such as 'acc', 'top'
             .etc will be inferred by 'greater' rule. Keys contain 'loss' will
@@ -239,15 +238,14 @@ class DistEvalHook(EvalHook):
         by_epoch (bool): Determine perform evaluation by epoch or by iteration.
             If set to True, it will perform by epoch. Otherwise, by iteration.
             default: True.
-        save_best (str | None, optional): If a metric is specified, it would
-            measure the best checkpoint during evaluation. The information
-            about best checkpoint would be save in best.json.
+        save_best (str, optional): If a metric is specified, it would measure
+            the best checkpoint during evaluation. The information about best
+            checkpoint would be save in best.json.
             Options are the evaluation metrics to the test dataset. e.g.,
-             ``top1_acc``, ``top5_acc``, ``mean_class_accuracy``,
-            ``mean_average_precision``, ``mmit_mean_average_precision``
-            for action recognition dataset (RawframeDataset and VideoDataset).
-            ``AR@AN``, ``auc`` for action localization dataset.
-            (ActivityNetDataset). Default: None.
+            ``bbox_mAP``, ``segm_mAP`` for bbox detection and instance
+            segmentation. ``AR@100`` for proposal recall. If ``save_best`` is
+            ``auto``, the first key will be used. The interval of
+            ``CheckpointHook`` should device EvalHook. Default: None.
         rule (str | None, optional): Comparison rule for best score. If set to
             None, it will infer a reasonable rule. Keys such as 'acc', 'top'
             .etc will be inferred by 'greater' rule. Keys contain 'loss' will

From ecabe8a0e8dd123240ec8530dc0b4d0a238d9897 Mon Sep 17 00:00:00 2001
From: dreamerlin <528557675@qq.com>
Date: Fri, 25 Dec 2020 17:08:56 +0800
Subject: [PATCH 04/17] polish

---
 mmcv/runner/hooks/eval.py | 30 +++++++++++++++++++-----------
 1 file changed, 19 insertions(+), 11 deletions(-)

diff --git a/mmcv/runner/hooks/eval.py b/mmcv/runner/hooks/eval.py
index 114406aee6..440372b52b 100644
--- a/mmcv/runner/hooks/eval.py
+++ b/mmcv/runner/hooks/eval.py
@@ -11,11 +11,14 @@
 
 class EvalHook(Hook):
     """Non-Distributed evaluation hook.
+
     Notes:
         If new arguments are added for EvalHook, tools/test.py,
         tools/eval_metric.py may be effected.
+
     This hook will regularly perform evaluation in a given interval when
     performing in non-distributed environment.
+
     Args:
         dataloader (DataLoader): A PyTorch dataloader.
         start (int | None, optional): Evaluation starting epoch. It enables
@@ -32,8 +35,9 @@ class EvalHook(Hook):
             Options are the evaluation metrics to the test dataset. e.g.,
             ``bbox_mAP``, ``segm_mAP`` for bbox detection and instance
             segmentation. ``AR@100`` for proposal recall. If ``save_best`` is
-            ``auto``, the first key will be used. The interval of
-            ``CheckpointHook`` should device EvalHook. Default: None.
+            ``auto``, the first key of the returned ``OrderedDict`` result
+            will be used. The interval of ``CheckpointHook`` should device
+            ``EvalHook``. Default: None.
         rule (str | None, optional): Comparison rule for best score. If set to
             None, it will infer a reasonable rule. Keys such as 'acc', 'top'
             .etc will be inferred by 'greater' rule. Keys contain 'loss' will
@@ -78,7 +82,7 @@ def __init__(self,
         assert isinstance(save_best, str) or save_best is None
         self.save_best = save_best
         self.eval_kwargs = eval_kwargs
-        self.initial_epoch_flag = True
+        self.initial_flag = True
 
         if self.save_best is not None:
             self._init_rule(rule, self.save_best)
@@ -121,21 +125,21 @@ def before_train_iter(self, runner):
         """Evaluate the model only at the start of training by iteration."""
         if self.by_epoch:
             return
-        if not self.initial_epoch_flag:
+        if not self.initial_flag:
             return
         if self.start is not None and runner.iter >= self.start:
             self.after_train_iter(runner)
-        self.initial_epoch_flag = False
+        self.initial_flag = False
 
     def before_train_epoch(self, runner):
         """Evaluate the model only at the start of training by epoch."""
         if not self.by_epoch:
             return
-        if not self.initial_epoch_flag:
+        if not self.initial_flag:
             return
         if self.start is not None and runner.epoch >= self.start:
             self.after_train_epoch(runner)
-        self.initial_epoch_flag = False
+        self.initial_flag = False
 
     def after_train_iter(self, runner):
         """Called after every training iter to evaluate the results."""
@@ -178,7 +182,8 @@ def evaluation_flag(self, runner):
             # No evaluation if start is larger than the current time.
             return False
         else:
-            # Evaluation only at epochs 3, 5, 7... if start==3 and interval==2
+            # Evaluation only at epochs/iters 3, 5, 7...
+            # if start==3 and interval==2
             if (current + 1 - self.start) % self.interval:
                 return False
         return True
@@ -187,7 +192,7 @@ def _save_ckpt(self, runner, key_score):
         if self.by_epoch:
             current = f'epoch_{runner.epoch + 1}'
         else:
-            current = f'iter_{runner.epoch + 1}'
+            current = f'iter_{runner.iter + 1}'
 
         best_score = runner.meta['hook_msgs'].get(
             'best_score', self.init_value_map[self.rule])
@@ -226,8 +231,10 @@ def evaluate(self, runner, results):
 
 class DistEvalHook(EvalHook):
     """Distributed evaluation hook.
+
     This hook will regularly perform evaluation in a given interval when
     performing in distributed environment.
+
     Args:
         dataloader (DataLoader): A PyTorch dataloader.
         start (int | None, optional): Evaluation starting epoch. It enables
@@ -244,8 +251,9 @@ class DistEvalHook(EvalHook):
             Options are the evaluation metrics to the test dataset. e.g.,
             ``bbox_mAP``, ``segm_mAP`` for bbox detection and instance
             segmentation. ``AR@100`` for proposal recall. If ``save_best`` is
-            ``auto``, the first key will be used. The interval of
-            ``CheckpointHook`` should device EvalHook. Default: None.
+            ``auto``, the first key of the returned ``OrderedDict`` result
+            will be used. The interval of ``CheckpointHook`` should depend on
+            ``EvalHook``. Default: None.
         rule (str | None, optional): Comparison rule for best score. If set to
             None, it will infer a reasonable rule. Keys such as 'acc', 'top'
             .etc will be inferred by 'greater' rule. Keys contain 'loss' will

From 3dd9d42918f21d44cd3fc36fb88037538865c985 Mon Sep 17 00:00:00 2001
From: dreamerlin <528557675@qq.com>
Date: Sat, 9 Jan 2021 20:48:47 +0800
Subject: [PATCH 05/17] use cp instead symlink

---
 mmcv/runner/hooks/eval.py           | 24 +++++++++++++++++-------
 tests/test_runner/test_eval_hook.py | 12 ++++++------
 2 files changed, 23 insertions(+), 13 deletions(-)

diff --git a/mmcv/runner/hooks/eval.py b/mmcv/runner/hooks/eval.py
index 440372b52b..d2f911b536 100644
--- a/mmcv/runner/hooks/eval.py
+++ b/mmcv/runner/hooks/eval.py
@@ -1,10 +1,10 @@
+import os
 import os.path as osp
 import warnings
 from math import inf
 
 from torch.utils.data import DataLoader
 
-from mmcv import symlink
 from mmcv.engine import multi_gpu_test, single_gpu_test
 from mmcv.runner import Hook
 
@@ -49,7 +49,7 @@ class EvalHook(Hook):
 
     rule_map = {'greater': lambda x, y: x > y, 'less': lambda x, y: x < y}
     init_value_map = {'greater': -inf, 'less': inf}
-    greater_keys = ['acc', 'top', 'AR@', 'auc', 'precision']
+    greater_keys = ['acc', 'top', 'AR@', 'auc', 'precision', 'mAP']
     less_keys = ['loss']
 
     def __init__(self,
@@ -85,6 +85,7 @@ def __init__(self,
         self.initial_flag = True
 
         if self.save_best is not None:
+            self.best_ckpt_path = None
             self._init_rule(rule, self.save_best)
 
     def _init_rule(self, rule, key_indicator):
@@ -191,8 +192,10 @@ def evaluation_flag(self, runner):
     def _save_ckpt(self, runner, key_score):
         if self.by_epoch:
             current = f'epoch_{runner.epoch + 1}'
+            cur_type, cur_time = 'epoch', runner.epoch + 1
         else:
             current = f'iter_{runner.iter + 1}'
+            cur_type, cur_time = 'iter', runner.iter + 1
 
         best_score = runner.meta['hook_msgs'].get(
             'best_score', self.init_value_map[self.rule])
@@ -201,12 +204,19 @@ def _save_ckpt(self, runner, key_score):
             runner.meta['hook_msgs']['best_score'] = best_score
             last_ckpt = runner.meta['hook_msgs']['last_ckpt']
             runner.meta['hook_msgs']['best_ckpt'] = last_ckpt
-            symlink(
-                last_ckpt,
-                osp.join(runner.work_dir, f'best_{self.key_indicator}.pth'))
+
+            if self.best_ckpt_path and osp.isfile(self.best_ckpt_path):
+                os.remove(self.best_ckpt_path)
+
+            best_ckpt_name = f'best_{self.key_indicator}_{current}.pth'
+            runner.save_checkpoint(
+                runner.work_dir, best_ckpt_name, create_symlink=False)
+            self.best_ckpt_path = osp.join(runner.work_dir, best_ckpt_name)
+            runner.logger.info(
+                f'Now best checkpoint is saved as {best_ckpt_name}.')
             runner.logger.info(
-                f'Now best checkpoint is {current}.pth.'
-                f'Best {self.key_indicator} is {best_score:0.4f}')
+                f'Best {self.key_indicator} is {best_score:0.4f} '
+                f'at {cur_time} {cur_type}.')
 
     def evaluate(self, runner, results):
         """Evaluate the results.
diff --git a/tests/test_runner/test_eval_hook.py b/tests/test_runner/test_eval_hook.py
index b359942bb2..6456895637 100644
--- a/tests/test_runner/test_eval_hook.py
+++ b/tests/test_runner/test_eval_hook.py
@@ -139,7 +139,7 @@ def test_eval_hook():
             runner.run([loader], [('train', 1)], 8)
 
             real_path = osp.join(tmpdir, 'epoch_4.pth')
-            link_path = osp.join(tmpdir, 'best_acc.pth')
+            link_path = osp.join(tmpdir, 'best_acc_epoch_4.pth')
 
             assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(
                 real_path)
@@ -161,7 +161,7 @@ def test_eval_hook():
             runner.run([loader], [('train', 1)], 8)
 
             real_path = osp.join(tmpdir, 'epoch_4.pth')
-            link_path = osp.join(tmpdir, 'best_acc.pth')
+            link_path = osp.join(tmpdir, 'best_acc_epoch_4.pth')
 
             assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(
                 real_path)
@@ -181,7 +181,7 @@ def test_eval_hook():
             runner.run([loader], [('train', 1)], 8)
 
             real_path = osp.join(tmpdir, 'epoch_4.pth')
-            link_path = osp.join(tmpdir, 'best_score.pth')
+            link_path = osp.join(tmpdir, 'best_score_epoch_4.pth')
 
             assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(
                 real_path)
@@ -201,7 +201,7 @@ def test_eval_hook():
             runner.run([loader], [('train', 1)], 8)
 
             real_path = osp.join(tmpdir, 'epoch_6.pth')
-            link_path = osp.join(tmpdir, 'best_acc.pth')
+            link_path = osp.join(tmpdir, 'best_acc_epoch_6.pth')
 
             assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(
                 real_path)
@@ -220,7 +220,7 @@ def test_eval_hook():
             runner.run([loader], [('train', 1)], 2)
 
             real_path = osp.join(tmpdir, 'epoch_2.pth')
-            link_path = osp.join(tmpdir, 'best_acc.pth')
+            link_path = osp.join(tmpdir, 'best_acc_epoch_2.pth')
 
             assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(
                 real_path)
@@ -238,7 +238,7 @@ def test_eval_hook():
             runner.run([loader], [('train', 1)], 8)
 
             real_path = osp.join(tmpdir, 'epoch_4.pth')
-            link_path = osp.join(tmpdir, 'best_acc.pth')
+            link_path = osp.join(tmpdir, 'best_acc_epoch_4.pth')
 
             assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(
                 real_path)

From 32e8410294801d1aed7a7e2aa5e16c5ec5c4e8e8 Mon Sep 17 00:00:00 2001
From: dreamerlin <528557675@qq.com>
Date: Sat, 9 Jan 2021 21:26:32 +0800
Subject: [PATCH 06/17] add error info

---
 mmcv/runner/hooks/eval.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/mmcv/runner/hooks/eval.py b/mmcv/runner/hooks/eval.py
index d2f911b536..02471d4555 100644
--- a/mmcv/runner/hooks/eval.py
+++ b/mmcv/runner/hooks/eval.py
@@ -67,19 +67,20 @@ def __init__(self,
         if interval <= 0:
             raise ValueError(f'interval must be positive, but got {interval}')
 
-        assert isinstance(by_epoch, bool)
+        assert isinstance(by_epoch, bool), '``by_epoch`` should be a boolean'
 
         if start is not None and start < 0:
             warnings.warn(
                 f'The evaluation start epoch {start} is smaller than 0, '
-                f'use 0 instead', UserWarning)
+                'use 0 instead', UserWarning)
             start = 0
         self.dataloader = dataloader
         self.interval = interval
         self.start = start
         self.by_epoch = by_epoch
 
-        assert isinstance(save_best, str) or save_best is None
+        assert isinstance(save_best, str) or save_best is None, \
+            '``save_best`` should be a str or None'
         self.save_best = save_best
         self.eval_kwargs = eval_kwargs
         self.initial_flag = True

From 0df56f1515bf6c531522ea4311be100bbc454d54 Mon Sep 17 00:00:00 2001
From: Jintao Lin <528557675@qq.com>
Date: Fri, 15 Jan 2021 16:57:39 +0800
Subject: [PATCH 07/17] Update eval.py

---
 mmcv/runner/hooks/eval.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mmcv/runner/hooks/eval.py b/mmcv/runner/hooks/eval.py
index 02471d4555..1edd12ff2a 100644
--- a/mmcv/runner/hooks/eval.py
+++ b/mmcv/runner/hooks/eval.py
@@ -203,8 +203,6 @@ def _save_ckpt(self, runner, key_score):
         if self.compare_func(key_score, best_score):
             best_score = key_score
             runner.meta['hook_msgs']['best_score'] = best_score
-            last_ckpt = runner.meta['hook_msgs']['last_ckpt']
-            runner.meta['hook_msgs']['best_ckpt'] = last_ckpt
 
             if self.best_ckpt_path and osp.isfile(self.best_ckpt_path):
                 os.remove(self.best_ckpt_path)
@@ -213,6 +211,8 @@ def _save_ckpt(self, runner, key_score):
             runner.save_checkpoint(
                 runner.work_dir, best_ckpt_name, create_symlink=False)
             self.best_ckpt_path = osp.join(runner.work_dir, best_ckpt_name)
+            
+            runner.meta['hook_msgs']['best_ckpt'] = self.best_ckpt_path
             runner.logger.info(
                 f'Now best checkpoint is saved as {best_ckpt_name}.')
             runner.logger.info(

From 0215744d8df25664fbd44fbfaa962eb67f6ee579 Mon Sep 17 00:00:00 2001
From: Jintao Lin <528557675@qq.com>
Date: Fri, 15 Jan 2021 17:01:47 +0800
Subject: [PATCH 08/17] Update eval.py

---
 mmcv/runner/hooks/eval.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/mmcv/runner/hooks/eval.py b/mmcv/runner/hooks/eval.py
index 1edd12ff2a..9e6bbff56f 100644
--- a/mmcv/runner/hooks/eval.py
+++ b/mmcv/runner/hooks/eval.py
@@ -211,7 +211,6 @@ def _save_ckpt(self, runner, key_score):
             runner.save_checkpoint(
                 runner.work_dir, best_ckpt_name, create_symlink=False)
             self.best_ckpt_path = osp.join(runner.work_dir, best_ckpt_name)
-            
             runner.meta['hook_msgs']['best_ckpt'] = self.best_ckpt_path
             runner.logger.info(
                 f'Now best checkpoint is saved as {best_ckpt_name}.')

From f0153cbf2d15a25f7315a7726975b0328fb6be1e Mon Sep 17 00:00:00 2001
From: dreamerlin <528557675@qq.com>
Date: Tue, 26 Jan 2021 01:15:28 +0800
Subject: [PATCH 09/17] update

---
 mmcv/runner/hooks/eval.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/mmcv/runner/hooks/eval.py b/mmcv/runner/hooks/eval.py
index 9e6bbff56f..92c920e89b 100644
--- a/mmcv/runner/hooks/eval.py
+++ b/mmcv/runner/hooks/eval.py
@@ -31,13 +31,13 @@ class EvalHook(Hook):
             default: True.
         save_best (str, optional): If a metric is specified, it would measure
             the best checkpoint during evaluation. The information about best
-            checkpoint would be save in best.json.
+            checkpoint would be save in ``runner.meta['hook_msgs']``.
             Options are the evaluation metrics to the test dataset. e.g.,
             ``bbox_mAP``, ``segm_mAP`` for bbox detection and instance
             segmentation. ``AR@100`` for proposal recall. If ``save_best`` is
             ``auto``, the first key of the returned ``OrderedDict`` result
-            will be used. The interval of ``CheckpointHook`` should device
-            ``EvalHook``. Default: None.
+            will be used. The interval of ``EvalHook`` should be
+            divisible of that in ``CheckpointHook``. Default: None.
         rule (str | None, optional): Comparison rule for best score. If set to
             None, it will infer a reasonable rule. Keys such as 'acc', 'top'
             .etc will be inferred by 'greater' rule. Keys contain 'loss' will
@@ -47,6 +47,10 @@ class EvalHook(Hook):
             the dataset.
     """
 
+    # Since the key for determine greater an less is related to the downstream
+    # task, downstream repos may need to overwrite the following inner
+    # variable according to their tasks
+
     rule_map = {'greater': lambda x, y: x > y, 'less': lambda x, y: x < y}
     init_value_map = {'greater': -inf, 'less': inf}
     greater_keys = ['acc', 'top', 'AR@', 'auc', 'precision', 'mAP']
@@ -257,13 +261,13 @@ class DistEvalHook(EvalHook):
             default: True.
         save_best (str, optional): If a metric is specified, it would measure
             the best checkpoint during evaluation. The information about best
-            checkpoint would be save in best.json.
+            checkpoint would be save in ``runner.meta['hook_msgs']``.
             Options are the evaluation metrics to the test dataset. e.g.,
             ``bbox_mAP``, ``segm_mAP`` for bbox detection and instance
             segmentation. ``AR@100`` for proposal recall. If ``save_best`` is
             ``auto``, the first key of the returned ``OrderedDict`` result
-            will be used. The interval of ``CheckpointHook`` should depend on
-            ``EvalHook``. Default: None.
+            will be used. The interval of ``EvalHook`` should depend on
+            ``CheckpointHook``. Default: None.
         rule (str | None, optional): Comparison rule for best score. If set to
             None, it will infer a reasonable rule. Keys such as 'acc', 'top'
             .etc will be inferred by 'greater' rule. Keys contain 'loss' will

From 656faacc5c2b0ef9fb4581bf2c51da87a9e3715e Mon Sep 17 00:00:00 2001
From: dreamerlin <528557675@qq.com>
Date: Wed, 24 Feb 2021 20:38:07 +0800
Subject: [PATCH 10/17] add engine depandancy

---
 mmcv/engine/__init__.py |   7 ++
 mmcv/engine/test.py     | 159 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 166 insertions(+)
 create mode 100644 mmcv/engine/__init__.py
 create mode 100644 mmcv/engine/test.py

diff --git a/mmcv/engine/__init__.py b/mmcv/engine/__init__.py
new file mode 100644
index 0000000000..8bec565dfc
--- /dev/null
+++ b/mmcv/engine/__init__.py
@@ -0,0 +1,7 @@
+from .test import (collect_results_cpu, collect_results_gpu, multi_gpu_test,
+                   single_gpu_test)
+
+__all__ = [
+    'collect_results_cpu', 'collect_results_gpu', 'multi_gpu_test',
+    'single_gpu_test'
+]
diff --git a/mmcv/engine/test.py b/mmcv/engine/test.py
new file mode 100644
index 0000000000..c1c8cb75d7
--- /dev/null
+++ b/mmcv/engine/test.py
@@ -0,0 +1,159 @@
+import os.path as osp
+import pickle
+import shutil
+import tempfile
+import time
+
+import torch
+import torch.distributed as dist
+
+import mmcv
+from mmcv.runner import get_dist_info
+
+
+def single_gpu_test(model, data_loader):
+    """Test model with a single gpu.
+
+    This method tests model with a single gpu and displays test progress bar.
+
+    Args:
+        model (nn.Module): Model to be tested.
+        data_loader (nn.Dataloader): Pytorch data loader.
+
+    Returns:
+        list: The prediction results.
+    """
+    model.eval()
+    results = []
+    dataset = data_loader.dataset
+    prog_bar = mmcv.ProgressBar(len(dataset))
+    for data in data_loader:
+        with torch.no_grad():
+            result = model(return_loss=False, **data)
+        results.extend(result)
+
+        # use the first key as main key to calculate the batch size
+        batch_size = len(next(iter(data.values())))
+        for _ in range(batch_size):
+            prog_bar.update()
+    return results
+
+
+def multi_gpu_test(model, data_loader, tmpdir=None, gpu_collect=False):
+    """Test model with multiple gpus.
+
+    This method tests model with multiple gpus and collects the results
+    under two different modes: gpu and cpu modes. By setting 'gpu_collect=True'
+    it encodes results to gpu tensors and use gpu communication for results
+    collection. On cpu mode it saves the results on different gpus to 'tmpdir'
+    and collects them by the rank 0 worker.
+
+    Args:
+        model (nn.Module): Model to be tested.
+        data_loader (nn.Dataloader): Pytorch data loader.
+        tmpdir (str): Path of directory to save the temporary results from
+            different gpus under cpu mode.
+        gpu_collect (bool): Option to use either gpu or cpu to collect results.
+
+    Returns:
+        list: The prediction results.
+    """
+    model.eval()
+    results = []
+    dataset = data_loader.dataset
+    rank, world_size = get_dist_info()
+    if rank == 0:
+        prog_bar = mmcv.ProgressBar(len(dataset))
+    time.sleep(2)  # This line can prevent deadlock problem in some cases.
+    for i, data in enumerate(data_loader):
+        with torch.no_grad():
+            result = model(return_loss=False, **data)
+        results.extend(result)
+
+        if rank == 0:
+            batch_size = len(result)
+            for _ in range(batch_size * world_size):
+                prog_bar.update()
+
+    # collect results from all ranks
+    if gpu_collect:
+        results = collect_results_gpu(results, len(dataset))
+    else:
+        results = collect_results_cpu(results, len(dataset), tmpdir)
+    return results
+
+
+def collect_results_cpu(result_part, size, tmpdir=None):
+    rank, world_size = get_dist_info()
+    # create a tmp dir if it is not specified
+    if tmpdir is None:
+        MAX_LEN = 512
+        # 32 is whitespace
+        dir_tensor = torch.full((MAX_LEN, ),
+                                32,
+                                dtype=torch.uint8,
+                                device='cuda')
+        if rank == 0:
+            mmcv.mkdir_or_exist('.dist_test')
+            tmpdir = tempfile.mkdtemp(dir='.dist_test')
+            tmpdir = torch.tensor(
+                bytearray(tmpdir.encode()), dtype=torch.uint8, device='cuda')
+            dir_tensor[:len(tmpdir)] = tmpdir
+        dist.broadcast(dir_tensor, 0)
+        tmpdir = dir_tensor.cpu().numpy().tobytes().decode().rstrip()
+    else:
+        mmcv.mkdir_or_exist(tmpdir)
+    # dump the part result to the dir
+    mmcv.dump(result_part, osp.join(tmpdir, f'part_{rank}.pkl'))
+    dist.barrier()
+    # collect all parts
+    if rank != 0:
+        return None
+    else:
+        # load results of all parts from tmp dir
+        part_list = []
+        for i in range(world_size):
+            part_file = osp.join(tmpdir, f'part_{i}.pkl')
+            part_list.append(mmcv.load(part_file))
+        # sort the results
+        ordered_results = []
+        for res in zip(*part_list):
+            ordered_results.extend(list(res))
+        # the dataloader may pad some samples
+        ordered_results = ordered_results[:size]
+        # remove tmp dir
+        shutil.rmtree(tmpdir)
+        return ordered_results
+
+
+def collect_results_gpu(result_part, size):
+    rank, world_size = get_dist_info()
+    # dump result part to tensor with pickle
+    part_tensor = torch.tensor(
+        bytearray(pickle.dumps(result_part)), dtype=torch.uint8, device='cuda')
+    # gather all result part tensor shape
+    shape_tensor = torch.tensor(part_tensor.shape, device='cuda')
+    shape_list = [shape_tensor.clone() for _ in range(world_size)]
+    dist.all_gather(shape_list, shape_tensor)
+    # padding result part tensor to max length
+    shape_max = torch.tensor(shape_list).max()
+    part_send = torch.zeros(shape_max, dtype=torch.uint8, device='cuda')
+    part_send[:shape_tensor[0]] = part_tensor
+    part_recv_list = [
+        part_tensor.new_zeros(shape_max) for _ in range(world_size)
+    ]
+    # gather all result part
+    dist.all_gather(part_recv_list, part_send)
+
+    if rank == 0:
+        part_list = []
+        for recv, shape in zip(part_recv_list, shape_list):
+            part_list.append(
+                pickle.loads(recv[:shape[0]].cpu().numpy().tobytes()))
+        # sort the results
+        ordered_results = []
+        for res in zip(*part_list):
+            ordered_results.extend(list(res))
+        # the dataloader may pad some samples
+        ordered_results = ordered_results[:size]
+        return ordered_results

From 97fc044d26486e9e5d5b38bf217edc9a8b953624 Mon Sep 17 00:00:00 2001
From: dreamerlin <528557675@qq.com>
Date: Wed, 24 Feb 2021 22:16:40 +0800
Subject: [PATCH 11/17] add comments

---
 mmcv/runner/hooks/__init__.py                |  2 +-
 mmcv/runner/hooks/{eval.py => evaluation.py} | 96 +++++++++++++++-----
 2 files changed, 74 insertions(+), 24 deletions(-)
 rename mmcv/runner/hooks/{eval.py => evaluation.py} (77%)

diff --git a/mmcv/runner/hooks/__init__.py b/mmcv/runner/hooks/__init__.py
index 6ece21fcb9..334a47c29b 100644
--- a/mmcv/runner/hooks/__init__.py
+++ b/mmcv/runner/hooks/__init__.py
@@ -2,7 +2,7 @@
 from .checkpoint import CheckpointHook
 from .closure import ClosureHook
 from .ema import EMAHook
-from .eval import DistEvalHook, EvalHook
+from .evaluation import DistEvalHook, EvalHook
 from .hook import HOOKS, Hook
 from .iter_timer import IterTimerHook
 from .logger import (LoggerHook, MlflowLoggerHook, PaviLoggerHook,
diff --git a/mmcv/runner/hooks/eval.py b/mmcv/runner/hooks/evaluation.py
similarity index 77%
rename from mmcv/runner/hooks/eval.py
rename to mmcv/runner/hooks/evaluation.py
index 92c920e89b..e1a034f768 100644
--- a/mmcv/runner/hooks/eval.py
+++ b/mmcv/runner/hooks/evaluation.py
@@ -3,6 +3,8 @@
 import warnings
 from math import inf
 
+import torch.distributed as dist
+from torch.nn.modules.batchnorm import _BatchNorm
 from torch.utils.data import DataLoader
 
 from mmcv.engine import multi_gpu_test, single_gpu_test
@@ -12,15 +14,12 @@
 class EvalHook(Hook):
     """Non-Distributed evaluation hook.
 
-    Notes:
-        If new arguments are added for EvalHook, tools/test.py,
-        tools/eval_metric.py may be effected.
-
     This hook will regularly perform evaluation in a given interval when
     performing in non-distributed environment.
 
     Args:
-        dataloader (DataLoader): A PyTorch dataloader.
+        dataloader (DataLoader): A PyTorch dataloader, whose dataset has
+            implemented ``evaluate`` function.
         start (int | None, optional): Evaluation starting epoch. It enables
             evaluation before the training starts if ``start`` <= the resuming
             epoch. If None, whether to evaluate is merely decided by
@@ -45,6 +44,10 @@ class EvalHook(Hook):
             Default: None.
         **eval_kwargs: Evaluation arguments fed into the evaluate function of
             the dataset.
+
+    Notes:
+        If new arguments are added for EvalHook, tools/test.py,
+        tools/eval_metric.py may be effected.
     """
 
     # Since the key for determine greater an less is related to the downstream
@@ -69,15 +72,15 @@ def __init__(self,
                             f'but got {type(dataloader)}')
 
         if interval <= 0:
-            raise ValueError(f'interval must be positive, but got {interval}')
+            raise ValueError(f'interval must be a positive number, '
+                             f'but got {interval}')
 
         assert isinstance(by_epoch, bool), '``by_epoch`` should be a boolean'
 
         if start is not None and start < 0:
-            warnings.warn(
-                f'The evaluation start epoch {start} is smaller than 0, '
-                'use 0 instead', UserWarning)
-            start = 0
+            raise ValueError(f'The evaluation start epoch {start} is smaller '
+                             f'than 0')
+
         self.dataloader = dataloader
         self.interval = interval
         self.start = start
@@ -96,6 +99,17 @@ def __init__(self,
     def _init_rule(self, rule, key_indicator):
         """Initialize rule, key_indicator, comparison_func, and best score.
 
+        Here is the rule to determine which rule is used for key indicator
+        when the rule is not specific:
+        1. If the key indicator is in ``self.greater_keys``, the rule will be
+           specified as 'greater'.
+        2. Or if the key indicator is in ``self.less_keys``, the rule will be
+           specified as 'less'.
+        3. Or if the key indicator is equal to the substring in any one item
+           in ``self.greater_keys``, the rule will be specified as 'greater'.
+        4. Or if the key indicator is equal to the substring in any one item
+           in ``self.less_keys``, the rule will be specified as 'less'.
+
         Args:
             rule (str | None): Comparison rule for best score.
             key_indicator (str | None): Key indicator to determine the
@@ -107,7 +121,11 @@ def _init_rule(self, rule, key_indicator):
 
         if rule is None:
             if key_indicator != 'auto':
-                if any(key in key_indicator for key in self.greater_keys):
+                if key_indicator in self.greater_keys:
+                    rule = 'greater'
+                elif key_indicator in self.less_keys:
+                    rule = 'less'
+                elif any(key in key_indicator for key in self.greater_keys):
                     rule = 'greater'
                 elif any(key in key_indicator for key in self.less_keys):
                     rule = 'less'
@@ -123,15 +141,13 @@ def _init_rule(self, rule, key_indicator):
     def before_run(self, runner):
         if self.save_best is not None:
             if runner.meta is None:
-                warnings.warn('runner.meta is None. Creating a empty one.')
+                warnings.warn('runner.meta is None. Creating an empty one.')
                 runner.meta = dict()
             runner.meta.setdefault('hook_msgs', dict())
 
     def before_train_iter(self, runner):
         """Evaluate the model only at the start of training by iteration."""
-        if self.by_epoch:
-            return
-        if not self.initial_flag:
+        if self.by_epoch or not self.initial_flag:
             return
         if self.start is not None and runner.iter >= self.start:
             self.after_train_iter(runner)
@@ -139,9 +155,7 @@ def before_train_iter(self, runner):
 
     def before_train_epoch(self, runner):
         """Evaluate the model only at the start of training by epoch."""
-        if not self.by_epoch:
-            return
-        if not self.initial_flag:
+        if not (self.by_epoch and self.initial_flag):
             return
         if self.start is not None and runner.epoch >= self.start:
             self.after_train_epoch(runner)
@@ -159,7 +173,7 @@ def after_train_epoch(self, runner):
 
     def _do_evaluate(self, runner):
         """perform evaluation and save ckpt."""
-        if not self.evaluation_flag(runner):
+        if not self._should_evaluate(runner):
             return
 
         results = single_gpu_test(runner.model, self.dataloader)
@@ -167,8 +181,16 @@ def _do_evaluate(self, runner):
         if self.save_best:
             self._save_ckpt(runner, key_score)
 
-    def evaluation_flag(self, runner):
-        """Judge whether to perform_evaluation.
+    def _should_evaluate(self, runner):
+        """Judge whether to perform evaluation.
+
+        Here is the rule to judge whether to perform evaluation:
+        1. It will not perform evaluation during the epoch/iteration interval,
+           which is determined by ``self.interval``.
+        2. It will not perform evaluation if the start time is larger than
+           current time.
+        3. It will not perform evaluation when current time is larger than
+           the start time but during epoch/iteration interval.
 
         Returns:
             bool: The flag indicating whether to perform evaluation.
@@ -195,6 +217,12 @@ def evaluation_flag(self, runner):
         return True
 
     def _save_ckpt(self, runner, key_score):
+        """Save the best checkpoint.
+
+        It will compare the score according to the compare function, write
+        related information (best score, best checkpoint path) and save the
+        best checkpoint into ``work_dir``.
+        """
         if self.by_epoch:
             current = f'epoch_{runner.epoch + 1}'
             cur_type, cur_time = 'epoch', runner.epoch + 1
@@ -234,6 +262,7 @@ def evaluate(self, runner, results):
         for name, val in eval_res.items():
             runner.log_buffer.output[name] = val
         runner.log_buffer.ready = True
+
         if self.save_best is not None:
             if self.key_indicator == 'auto':
                 # infer from eval_results
@@ -250,7 +279,8 @@ class DistEvalHook(EvalHook):
     performing in distributed environment.
 
     Args:
-        dataloader (DataLoader): A PyTorch dataloader.
+        dataloader (DataLoader): A PyTorch dataloader, whose dataset has
+            implemented ``evaluate`` function.
         start (int | None, optional): Evaluation starting epoch. It enables
             evaluation before the training starts if ``start`` <= the resuming
             epoch. If None, whether to evaluate is merely decided by
@@ -277,6 +307,9 @@ class DistEvalHook(EvalHook):
             processes. Default: None.
         gpu_collect (bool): Whether to use gpu or cpu to collect results.
             Default: False.
+        broadcast_bn_buffer (bool): Whether to broadcast the
+            buffer(running_mean and running_var) of rank 0 to other rank
+            before evaluation. Default: True.
         **eval_kwargs: Evaluation arguments fed into the evaluate function of
             the dataset.
     """
@@ -288,6 +321,7 @@ def __init__(self,
                  by_epoch=True,
                  save_best=None,
                  rule=None,
+                 broadcast_bn_buffer=True,
                  tmpdir=None,
                  gpu_collect=False,
                  **eval_kwargs):
@@ -299,11 +333,27 @@ def __init__(self,
             save_best=save_best,
             rule=rule,
             **eval_kwargs)
+        self.broadcast_bn_buffer = broadcast_bn_buffer
         self.tmpdir = tmpdir
         self.gpu_collect = gpu_collect
 
     def _do_evaluate(self, runner):
-        if not self.evaluation_flag(runner):
+        """perform evaluation and save ckpt."""
+
+        # Synchronization of BatchNorm's buffer (running_mean
+        # and running_var) is not supported in the DDP of pytorch,
+        # which may cause the inconsistent performance of models in
+        # different ranks, so we broadcast BatchNorm's buffers
+        # of rank 0 to other ranks to avoid this.
+        if self.broadcast_bn_buffer:
+            model = runner.model
+            for name, module in model.named_modules():
+                if isinstance(module,
+                              _BatchNorm) and module.track_running_stats:
+                    dist.broadcast(module.running_var, 0)
+                    dist.broadcast(module.running_mean, 0)
+
+        if not self._should_evaluate(runner):
             return
 
         tmpdir = self.tmpdir

From 89d514810fa6e6bcd141b2bc7dba586a9f45723b Mon Sep 17 00:00:00 2001
From: dreamerlin <528557675@qq.com>
Date: Wed, 24 Feb 2021 23:00:14 +0800
Subject: [PATCH 12/17] fix unittest

---
 mmcv/runner/hooks/evaluation.py     |  5 +--
 tests/test_runner/test_eval_hook.py | 56 ++++++++++++-----------------
 2 files changed, 26 insertions(+), 35 deletions(-)

diff --git a/mmcv/runner/hooks/evaluation.py b/mmcv/runner/hooks/evaluation.py
index e1a034f768..72a57be940 100644
--- a/mmcv/runner/hooks/evaluation.py
+++ b/mmcv/runner/hooks/evaluation.py
@@ -7,8 +7,7 @@
 from torch.nn.modules.batchnorm import _BatchNorm
 from torch.utils.data import DataLoader
 
-from mmcv.engine import multi_gpu_test, single_gpu_test
-from mmcv.runner import Hook
+from .hook import Hook
 
 
 class EvalHook(Hook):
@@ -176,6 +175,7 @@ def _do_evaluate(self, runner):
         if not self._should_evaluate(runner):
             return
 
+        from mmcv.engine import single_gpu_test
         results = single_gpu_test(runner.model, self.dataloader)
         key_score = self.evaluate(runner, results)
         if self.save_best:
@@ -360,6 +360,7 @@ def _do_evaluate(self, runner):
         if tmpdir is None:
             tmpdir = osp.join(runner.work_dir, '.eval_hook')
 
+        from mmcv.engine import multi_gpu_test
         results = multi_gpu_test(
             runner.model,
             self.dataloader,
diff --git a/tests/test_runner/test_eval_hook.py b/tests/test_runner/test_eval_hook.py
index 6456895637..df45c14761 100644
--- a/tests/test_runner/test_eval_hook.py
+++ b/tests/test_runner/test_eval_hook.py
@@ -138,12 +138,11 @@ def test_eval_hook():
             runner.register_hook(eval_hook)
             runner.run([loader], [('train', 1)], 8)
 
-            real_path = osp.join(tmpdir, 'epoch_4.pth')
-            link_path = osp.join(tmpdir, 'best_acc_epoch_4.pth')
+            ckpt_path = osp.join(tmpdir, 'best_acc_epoch_4.pth')
 
             assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(
-                real_path)
-            assert osp.exists(link_path)
+                ckpt_path)
+            assert osp.exists(ckpt_path)
             assert runner.meta['hook_msgs']['best_score'] == 7
 
         # total_epochs = 8, return the best acc and corresponding epoch
@@ -160,12 +159,11 @@ def test_eval_hook():
             runner.register_hook(eval_hook)
             runner.run([loader], [('train', 1)], 8)
 
-            real_path = osp.join(tmpdir, 'epoch_4.pth')
-            link_path = osp.join(tmpdir, 'best_acc_epoch_4.pth')
+            ckpt_path = osp.join(tmpdir, 'best_acc_epoch_4.pth')
 
             assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(
-                real_path)
-            assert osp.exists(link_path)
+                ckpt_path)
+            assert osp.exists(ckpt_path)
             assert runner.meta['hook_msgs']['best_score'] == 7
 
         # total_epochs = 8, return the best score and corresponding epoch
@@ -180,12 +178,11 @@ def test_eval_hook():
             runner.register_hook(eval_hook)
             runner.run([loader], [('train', 1)], 8)
 
-            real_path = osp.join(tmpdir, 'epoch_4.pth')
-            link_path = osp.join(tmpdir, 'best_score_epoch_4.pth')
+            ckpt_path = osp.join(tmpdir, 'best_score_epoch_4.pth')
 
             assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(
-                real_path)
-            assert osp.exists(link_path)
+                ckpt_path)
+            assert osp.exists(ckpt_path)
             assert runner.meta['hook_msgs']['best_score'] == 7
 
         # total_epochs = 8, return the best score using less compare func
@@ -200,12 +197,11 @@ def test_eval_hook():
             runner.register_hook(eval_hook)
             runner.run([loader], [('train', 1)], 8)
 
-            real_path = osp.join(tmpdir, 'epoch_6.pth')
-            link_path = osp.join(tmpdir, 'best_acc_epoch_6.pth')
+            ckpt_path = osp.join(tmpdir, 'best_acc_epoch_6.pth')
 
             assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(
-                real_path)
-            assert osp.exists(link_path)
+                ckpt_path)
+            assert osp.exists(ckpt_path)
             assert runner.meta['hook_msgs']['best_score'] == -3
 
         # Test the EvalHook when resume happend
@@ -219,12 +215,11 @@ def test_eval_hook():
             runner.register_hook(eval_hook)
             runner.run([loader], [('train', 1)], 2)
 
-            real_path = osp.join(tmpdir, 'epoch_2.pth')
-            link_path = osp.join(tmpdir, 'best_acc_epoch_2.pth')
+            ckpt_path = osp.join(tmpdir, 'best_acc_epoch_2.pth')
 
             assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(
-                real_path)
-            assert osp.exists(link_path)
+                ckpt_path)
+            assert osp.exists(ckpt_path)
             assert runner.meta['hook_msgs']['best_score'] == 4
 
             resume_from = osp.join(tmpdir, 'latest.pth')
@@ -237,12 +232,11 @@ def test_eval_hook():
             runner.resume(resume_from)
             runner.run([loader], [('train', 1)], 8)
 
-            real_path = osp.join(tmpdir, 'epoch_4.pth')
-            link_path = osp.join(tmpdir, 'best_acc_epoch_4.pth')
+            ckpt_path = osp.join(tmpdir, 'best_acc_epoch_4.pth')
 
             assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(
-                real_path)
-            assert osp.exists(link_path)
+                ckpt_path)
+            assert osp.exists(ckpt_path)
             assert runner.meta['hook_msgs']['best_score'] == 7
 
 
@@ -264,6 +258,10 @@ def test_start_param(EvalHookParam, _build_demo_runner, by_epoch):
     with pytest.raises(ValueError):
         EvalHookParam(dataloader, interval=-1)
 
+    # 0.3. negative start
+    with pytest.raises(ValueError):
+        EvalHookParam(dataloader, start=-1)
+
     # 1. start=None, interval=1: perform evaluation after each epoch.
     runner = _build_demo_runner()
     evalhook = EvalHookParam(dataloader, interval=1, by_epoch=by_epoch)
@@ -298,7 +296,7 @@ def test_start_param(EvalHookParam, _build_demo_runner, by_epoch):
     runner.run([dataloader], [('train', 1)], 3)
     assert evalhook.evaluate.call_count == 2  # after epoch 1 & 3
 
-    # 5. start=0/negative, interval=1: perform evaluation after each epoch and
+    # 5. start=0, interval=1: perform evaluation after each epoch and
     #    before epoch 1.
     runner = _build_demo_runner()
     evalhook = EvalHookParam(dataloader, start=0, by_epoch=by_epoch)
@@ -307,14 +305,6 @@ def test_start_param(EvalHookParam, _build_demo_runner, by_epoch):
     runner.run([dataloader], [('train', 1)], 2)
     assert evalhook.evaluate.call_count == 3  # before epoch1 and after e1 & e2
 
-    runner = _build_demo_runner()
-    with pytest.warns(UserWarning):
-        evalhook = EvalHookParam(dataloader, start=-2, by_epoch=by_epoch)
-    evalhook.evaluate = MagicMock()
-    runner.register_hook(evalhook)
-    runner.run([dataloader], [('train', 1)], 2)
-    assert evalhook.evaluate.call_count == 3  # before epoch1 and after e1 & e2
-
     # 6. resuming from epoch i, start = x (x<=i), interval =1: perform
     #    evaluation after each epoch and before the first epoch.
     runner = _build_demo_runner()

From da6ae645a4e04315e934eb97aeaac19dcbe33391 Mon Sep 17 00:00:00 2001
From: dreamerlin <528557675@qq.com>
Date: Wed, 24 Feb 2021 23:18:58 +0800
Subject: [PATCH 13/17] fix

---
 tests/test_runner/test_eval_hook.py | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/tests/test_runner/test_eval_hook.py b/tests/test_runner/test_eval_hook.py
index df45c14761..6c9314b93f 100644
--- a/tests/test_runner/test_eval_hook.py
+++ b/tests/test_runner/test_eval_hook.py
@@ -140,8 +140,7 @@ def test_eval_hook():
 
             ckpt_path = osp.join(tmpdir, 'best_acc_epoch_4.pth')
 
-            assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(
-                ckpt_path)
+            assert runner.meta['hook_msgs']['best_ckpt'] == ckpt_path
             assert osp.exists(ckpt_path)
             assert runner.meta['hook_msgs']['best_score'] == 7
 
@@ -161,8 +160,7 @@ def test_eval_hook():
 
             ckpt_path = osp.join(tmpdir, 'best_acc_epoch_4.pth')
 
-            assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(
-                ckpt_path)
+            assert runner.meta['hook_msgs']['best_ckpt'] == ckpt_path
             assert osp.exists(ckpt_path)
             assert runner.meta['hook_msgs']['best_score'] == 7
 
@@ -180,8 +178,7 @@ def test_eval_hook():
 
             ckpt_path = osp.join(tmpdir, 'best_score_epoch_4.pth')
 
-            assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(
-                ckpt_path)
+            assert runner.meta['hook_msgs']['best_ckpt'] == ckpt_path
             assert osp.exists(ckpt_path)
             assert runner.meta['hook_msgs']['best_score'] == 7
 
@@ -199,8 +196,7 @@ def test_eval_hook():
 
             ckpt_path = osp.join(tmpdir, 'best_acc_epoch_6.pth')
 
-            assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(
-                ckpt_path)
+            assert runner.meta['hook_msgs']['best_ckpt'] == ckpt_path
             assert osp.exists(ckpt_path)
             assert runner.meta['hook_msgs']['best_score'] == -3
 
@@ -217,8 +213,7 @@ def test_eval_hook():
 
             ckpt_path = osp.join(tmpdir, 'best_acc_epoch_2.pth')
 
-            assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(
-                ckpt_path)
+            assert runner.meta['hook_msgs']['best_ckpt'] == ckpt_path
             assert osp.exists(ckpt_path)
             assert runner.meta['hook_msgs']['best_score'] == 4
 
@@ -234,8 +229,7 @@ def test_eval_hook():
 
             ckpt_path = osp.join(tmpdir, 'best_acc_epoch_4.pth')
 
-            assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(
-                ckpt_path)
+            assert runner.meta['hook_msgs']['best_ckpt'] == ckpt_path
             assert osp.exists(ckpt_path)
             assert runner.meta['hook_msgs']['best_score'] == 7
 

From 954811c36fc2dca690f39ae47f45f4d85c08a284 Mon Sep 17 00:00:00 2001
From: dreamerlin <528557675@qq.com>
Date: Thu, 25 Feb 2021 11:46:36 +0800
Subject: [PATCH 14/17] update unittest

---
 tests/test_runner/test_eval_hook.py | 254 ++++++++++++++++------------
 1 file changed, 144 insertions(+), 110 deletions(-)

diff --git a/tests/test_runner/test_eval_hook.py b/tests/test_runner/test_eval_hook.py
index 6c9314b93f..755efb5868 100644
--- a/tests/test_runner/test_eval_hook.py
+++ b/tests/test_runner/test_eval_hook.py
@@ -9,8 +9,10 @@
 import torch.nn as nn
 from torch.utils.data import DataLoader, Dataset
 
-from mmcv.runner import (DistEvalHook, EpochBasedRunner, EvalHook,
-                         IterBasedRunner)
+from mmcv.runner import DistEvalHook as BaseDistEvalHook
+from mmcv.runner import EpochBasedRunner
+from mmcv.runner import EvalHook as BaseEvalHook
+from mmcv.runner import IterBasedRunner
 from mmcv.utils import get_logger
 
 
@@ -36,7 +38,8 @@ class EvalDataset(ExampleDataset):
 
     def evaluate(self, results, logger=None):
         acc = self.eval_result[self.index]
-        output = OrderedDict(acc=acc, index=self.index, score=acc)
+        output = OrderedDict(
+            acc=acc, index=self.index, score=acc, loss_top=acc)
         self.index += 1
         return output
 
@@ -79,6 +82,24 @@ def _build_iter_runner():
     return runner
 
 
+class EvalHook(BaseEvalHook):
+
+    greater_keys = ['acc', 'top']
+    less_keys = ['loss', 'loss_top']
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+
+class DistEvalHook(BaseDistEvalHook):
+
+    greater_keys = ['acc', 'top']
+    less_keys = ['loss', 'loss_top']
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+
 def test_eval_hook():
     with pytest.raises(AssertionError):
         # `save_best` should be a str
@@ -124,114 +145,127 @@ def test_eval_hook():
         assert runner.meta is None or 'best_ckpt' not in runner.meta[
             'hook_msgs']
 
-        # when `save_best` is set to 'auto', first metric will be used.
-        loader = DataLoader(EvalDataset())
-        model = Model()
-        data_loader = DataLoader(EvalDataset())
-        eval_hook = EvalHook(data_loader, interval=1, save_best='auto')
-
-        with tempfile.TemporaryDirectory() as tmpdir:
-            logger = get_logger('test_eval')
-            runner = EpochBasedRunner(
-                model=model, work_dir=tmpdir, logger=logger)
-            runner.register_checkpoint_hook(dict(interval=1))
-            runner.register_hook(eval_hook)
-            runner.run([loader], [('train', 1)], 8)
-
-            ckpt_path = osp.join(tmpdir, 'best_acc_epoch_4.pth')
-
-            assert runner.meta['hook_msgs']['best_ckpt'] == ckpt_path
-            assert osp.exists(ckpt_path)
-            assert runner.meta['hook_msgs']['best_score'] == 7
-
-        # total_epochs = 8, return the best acc and corresponding epoch
-        loader = DataLoader(EvalDataset())
-        model = Model()
-        data_loader = DataLoader(EvalDataset())
-        eval_hook = EvalHook(data_loader, interval=1, save_best='acc')
-
-        with tempfile.TemporaryDirectory() as tmpdir:
-            logger = get_logger('test_eval')
-            runner = EpochBasedRunner(
-                model=model, work_dir=tmpdir, logger=logger)
-            runner.register_checkpoint_hook(dict(interval=1))
-            runner.register_hook(eval_hook)
-            runner.run([loader], [('train', 1)], 8)
-
-            ckpt_path = osp.join(tmpdir, 'best_acc_epoch_4.pth')
-
-            assert runner.meta['hook_msgs']['best_ckpt'] == ckpt_path
-            assert osp.exists(ckpt_path)
-            assert runner.meta['hook_msgs']['best_score'] == 7
-
-        # total_epochs = 8, return the best score and corresponding epoch
-        data_loader = DataLoader(EvalDataset())
-        eval_hook = EvalHook(
-            data_loader, interval=1, save_best='score', rule='greater')
-        with tempfile.TemporaryDirectory() as tmpdir:
-            logger = get_logger('test_eval')
-            runner = EpochBasedRunner(
-                model=model, work_dir=tmpdir, logger=logger)
-            runner.register_checkpoint_hook(dict(interval=1))
-            runner.register_hook(eval_hook)
-            runner.run([loader], [('train', 1)], 8)
-
-            ckpt_path = osp.join(tmpdir, 'best_score_epoch_4.pth')
-
-            assert runner.meta['hook_msgs']['best_ckpt'] == ckpt_path
-            assert osp.exists(ckpt_path)
-            assert runner.meta['hook_msgs']['best_score'] == 7
-
-        # total_epochs = 8, return the best score using less compare func
-        # and indicate corresponding epoch
-        data_loader = DataLoader(EvalDataset())
-        eval_hook = EvalHook(data_loader, save_best='acc', rule='less')
-        with tempfile.TemporaryDirectory() as tmpdir:
-            logger = get_logger('test_eval')
-            runner = EpochBasedRunner(
-                model=model, work_dir=tmpdir, logger=logger)
-            runner.register_checkpoint_hook(dict(interval=1))
-            runner.register_hook(eval_hook)
-            runner.run([loader], [('train', 1)], 8)
-
-            ckpt_path = osp.join(tmpdir, 'best_acc_epoch_6.pth')
-
-            assert runner.meta['hook_msgs']['best_ckpt'] == ckpt_path
-            assert osp.exists(ckpt_path)
-            assert runner.meta['hook_msgs']['best_score'] == -3
-
-        # Test the EvalHook when resume happend
-        data_loader = DataLoader(EvalDataset())
+    # when `save_best` is set to 'auto', first metric will be used.
+    loader = DataLoader(EvalDataset())
+    model = Model()
+    data_loader = DataLoader(EvalDataset())
+    eval_hook = EvalHook(data_loader, interval=1, save_best='auto')
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        logger = get_logger('test_eval')
+        runner = EpochBasedRunner(model=model, work_dir=tmpdir, logger=logger)
+        runner.register_checkpoint_hook(dict(interval=1))
+        runner.register_hook(eval_hook)
+        runner.run([loader], [('train', 1)], 8)
+
+        ckpt_path = osp.join(tmpdir, 'best_acc_epoch_4.pth')
+
+        assert runner.meta['hook_msgs']['best_ckpt'] == ckpt_path
+        assert osp.exists(ckpt_path)
+        assert runner.meta['hook_msgs']['best_score'] == 7
+
+    # total_epochs = 8, return the best acc and corresponding epoch
+    loader = DataLoader(EvalDataset())
+    model = Model()
+    data_loader = DataLoader(EvalDataset())
+    eval_hook = EvalHook(data_loader, interval=1, save_best='acc')
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        logger = get_logger('test_eval')
+        runner = EpochBasedRunner(model=model, work_dir=tmpdir, logger=logger)
+        runner.register_checkpoint_hook(dict(interval=1))
+        runner.register_hook(eval_hook)
+        runner.run([loader], [('train', 1)], 8)
+
+        ckpt_path = osp.join(tmpdir, 'best_acc_epoch_4.pth')
+
+        assert runner.meta['hook_msgs']['best_ckpt'] == ckpt_path
+        assert osp.exists(ckpt_path)
+        assert runner.meta['hook_msgs']['best_score'] == 7
+
+    # total_epochs = 8, return the best loss_top and corresponding epoch
+    loader = DataLoader(EvalDataset())
+    model = Model()
+    data_loader = DataLoader(EvalDataset())
+    eval_hook = EvalHook(data_loader, interval=1, save_best='loss_top')
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        logger = get_logger('test_eval')
+        runner = EpochBasedRunner(model=model, work_dir=tmpdir, logger=logger)
+        runner.register_checkpoint_hook(dict(interval=1))
+        runner.register_hook(eval_hook)
+        runner.run([loader], [('train', 1)], 8)
+
+        ckpt_path = osp.join(tmpdir, 'best_loss_top_epoch_6.pth')
+
+        assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(ckpt_path)
+        assert osp.exists(ckpt_path)
+        assert runner.meta['hook_msgs']['best_score'] == -3
+
+    # total_epochs = 8, return the best score and corresponding epoch
+    data_loader = DataLoader(EvalDataset())
+    eval_hook = EvalHook(
+        data_loader, interval=1, save_best='score', rule='greater')
+    with tempfile.TemporaryDirectory() as tmpdir:
+        logger = get_logger('test_eval')
+        runner = EpochBasedRunner(model=model, work_dir=tmpdir, logger=logger)
+        runner.register_checkpoint_hook(dict(interval=1))
+        runner.register_hook(eval_hook)
+        runner.run([loader], [('train', 1)], 8)
+
+        ckpt_path = osp.join(tmpdir, 'best_score_epoch_4.pth')
+
+        assert runner.meta['hook_msgs']['best_ckpt'] == ckpt_path
+        assert osp.exists(ckpt_path)
+        assert runner.meta['hook_msgs']['best_score'] == 7
+
+    # total_epochs = 8, return the best score using less compare func
+    # and indicate corresponding epoch
+    data_loader = DataLoader(EvalDataset())
+    eval_hook = EvalHook(data_loader, save_best='acc', rule='less')
+    with tempfile.TemporaryDirectory() as tmpdir:
+        logger = get_logger('test_eval')
+        runner = EpochBasedRunner(model=model, work_dir=tmpdir, logger=logger)
+        runner.register_checkpoint_hook(dict(interval=1))
+        runner.register_hook(eval_hook)
+        runner.run([loader], [('train', 1)], 8)
+
+        ckpt_path = osp.join(tmpdir, 'best_acc_epoch_6.pth')
+
+        assert runner.meta['hook_msgs']['best_ckpt'] == ckpt_path
+        assert osp.exists(ckpt_path)
+        assert runner.meta['hook_msgs']['best_score'] == -3
+
+    # Test the EvalHook when resume happend
+    data_loader = DataLoader(EvalDataset())
+    eval_hook = EvalHook(data_loader, save_best='acc')
+    with tempfile.TemporaryDirectory() as tmpdir:
+        logger = get_logger('test_eval')
+        runner = EpochBasedRunner(model=model, work_dir=tmpdir, logger=logger)
+        runner.register_checkpoint_hook(dict(interval=1))
+        runner.register_hook(eval_hook)
+        runner.run([loader], [('train', 1)], 2)
+
+        ckpt_path = osp.join(tmpdir, 'best_acc_epoch_2.pth')
+
+        assert runner.meta['hook_msgs']['best_ckpt'] == ckpt_path
+        assert osp.exists(ckpt_path)
+        assert runner.meta['hook_msgs']['best_score'] == 4
+
+        resume_from = osp.join(tmpdir, 'latest.pth')
+        loader = DataLoader(ExampleDataset())
         eval_hook = EvalHook(data_loader, save_best='acc')
-        with tempfile.TemporaryDirectory() as tmpdir:
-            logger = get_logger('test_eval')
-            runner = EpochBasedRunner(
-                model=model, work_dir=tmpdir, logger=logger)
-            runner.register_checkpoint_hook(dict(interval=1))
-            runner.register_hook(eval_hook)
-            runner.run([loader], [('train', 1)], 2)
-
-            ckpt_path = osp.join(tmpdir, 'best_acc_epoch_2.pth')
-
-            assert runner.meta['hook_msgs']['best_ckpt'] == ckpt_path
-            assert osp.exists(ckpt_path)
-            assert runner.meta['hook_msgs']['best_score'] == 4
-
-            resume_from = osp.join(tmpdir, 'latest.pth')
-            loader = DataLoader(ExampleDataset())
-            eval_hook = EvalHook(data_loader, save_best='acc')
-            runner = EpochBasedRunner(
-                model=model, work_dir=tmpdir, logger=logger)
-            runner.register_checkpoint_hook(dict(interval=1))
-            runner.register_hook(eval_hook)
-            runner.resume(resume_from)
-            runner.run([loader], [('train', 1)], 8)
-
-            ckpt_path = osp.join(tmpdir, 'best_acc_epoch_4.pth')
-
-            assert runner.meta['hook_msgs']['best_ckpt'] == ckpt_path
-            assert osp.exists(ckpt_path)
-            assert runner.meta['hook_msgs']['best_score'] == 7
+        runner = EpochBasedRunner(model=model, work_dir=tmpdir, logger=logger)
+        runner.register_checkpoint_hook(dict(interval=1))
+        runner.register_hook(eval_hook)
+        runner.resume(resume_from)
+        runner.run([loader], [('train', 1)], 8)
+
+        ckpt_path = osp.join(tmpdir, 'best_acc_epoch_4.pth')
+
+        assert runner.meta['hook_msgs']['best_ckpt'] == ckpt_path
+        assert osp.exists(ckpt_path)
+        assert runner.meta['hook_msgs']['best_score'] == 7
 
 
 @patch('mmcv.engine.single_gpu_test', MagicMock)

From f6a35d619dda7128c5e595f90811d23f04bd5ca6 Mon Sep 17 00:00:00 2001
From: dreamerlin <528557675@qq.com>
Date: Thu, 25 Feb 2021 17:33:15 +0800
Subject: [PATCH 15/17] update unittest

---
 tests/test_runner/test_eval_hook.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_runner/test_eval_hook.py b/tests/test_runner/test_eval_hook.py
index 755efb5868..2d9fe39cca 100644
--- a/tests/test_runner/test_eval_hook.py
+++ b/tests/test_runner/test_eval_hook.py
@@ -198,7 +198,7 @@ def test_eval_hook():
 
         ckpt_path = osp.join(tmpdir, 'best_loss_top_epoch_6.pth')
 
-        assert runner.meta['hook_msgs']['best_ckpt'] == osp.realpath(ckpt_path)
+        assert runner.meta['hook_msgs']['best_ckpt'] == ckpt_path
         assert osp.exists(ckpt_path)
         assert runner.meta['hook_msgs']['best_score'] == -3
 

From db3122bd0ef5397200d71dead953696c5f122809 Mon Sep 17 00:00:00 2001
From: dreamerlin <528557675@qq.com>
Date: Sun, 28 Feb 2021 21:21:09 +0800
Subject: [PATCH 16/17] add docstring

---
 mmcv/engine/test.py | 52 +++++++++++++++++++++++++++++++++++++++------
 1 file changed, 45 insertions(+), 7 deletions(-)

diff --git a/mmcv/engine/test.py b/mmcv/engine/test.py
index c1c8cb75d7..57b261c4c0 100644
--- a/mmcv/engine/test.py
+++ b/mmcv/engine/test.py
@@ -43,10 +43,10 @@ def multi_gpu_test(model, data_loader, tmpdir=None, gpu_collect=False):
     """Test model with multiple gpus.
 
     This method tests model with multiple gpus and collects the results
-    under two different modes: gpu and cpu modes. By setting 'gpu_collect=True'
-    it encodes results to gpu tensors and use gpu communication for results
-    collection. On cpu mode it saves the results on different gpus to 'tmpdir'
-    and collects them by the rank 0 worker.
+    under two different modes: gpu and cpu modes. By setting
+    ``gpu_collect=True``, it encodes results to gpu tensors and use gpu
+    communication for results collection. On cpu mode it saves the results on
+    different gpus to ``tmpdir`` and collects them by the rank 0 worker.
 
     Args:
         model (nn.Module): Model to be tested.
@@ -84,6 +84,23 @@ def multi_gpu_test(model, data_loader, tmpdir=None, gpu_collect=False):
 
 
 def collect_results_cpu(result_part, size, tmpdir=None):
+    """Collect results under cpu mode.
+
+    On cpu mode, this function will save the results on different gpus to
+    ``tmpdir`` and collect them by the rank 0 worker.
+
+    Args:
+        result_part (list): Result list containing result parts
+            to be collected.
+        size (int): Size of the results, commonly equal to length of
+            the results.
+        tmpdir (str | None): temporal directory for collected results to
+            store. If set to None, it will create a random temporal directory
+            for it.
+
+    Returns:
+        list: The collected results.
+    """
     rank, world_size = get_dist_info()
     # create a tmp dir if it is not specified
     if tmpdir is None:
@@ -114,7 +131,11 @@ def collect_results_cpu(result_part, size, tmpdir=None):
         part_list = []
         for i in range(world_size):
             part_file = osp.join(tmpdir, f'part_{i}.pkl')
-            part_list.append(mmcv.load(part_file))
+            part_result = mmcv.load(part_file)
+            # When data is severely insufficient, an empty part_result
+            # on a certain gpu could makes the overall outputs empty.
+            if part_result:
+                part_list.append(part_result)
         # sort the results
         ordered_results = []
         for res in zip(*part_list):
@@ -127,6 +148,20 @@ def collect_results_cpu(result_part, size, tmpdir=None):
 
 
 def collect_results_gpu(result_part, size):
+    """Collect results under gpu mode.
+
+    On gpu mode, this function will encode results to gpu tensors and use gpu
+    communication for results collection.
+
+    Args:
+        result_part (list): Result list containing result parts
+            to be collected.
+        size (int): Size of the results, commonly equal to length of
+            the results.
+
+    Returns:
+        list: The collected results.
+    """
     rank, world_size = get_dist_info()
     # dump result part to tensor with pickle
     part_tensor = torch.tensor(
@@ -148,8 +183,11 @@ def collect_results_gpu(result_part, size):
     if rank == 0:
         part_list = []
         for recv, shape in zip(part_recv_list, shape_list):
-            part_list.append(
-                pickle.loads(recv[:shape[0]].cpu().numpy().tobytes()))
+            part_result = pickle.loads(recv[:shape[0]].cpu().numpy().tobytes())
+            # When data is severely insufficient, an empty part_result
+            # on a certain gpu could makes the overall outputs empty.
+            if part_result:
+                part_list.append(part_result)
         # sort the results
         ordered_results = []
         for res in zip(*part_list):

From 10b8d62aa12ca3762597664252039e440e552876 Mon Sep 17 00:00:00 2001
From: dreamerlin <528557675@qq.com>
Date: Wed, 3 Mar 2021 19:39:15 +0800
Subject: [PATCH 17/17] fix docstring

---
 mmcv/runner/hooks/evaluation.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/mmcv/runner/hooks/evaluation.py b/mmcv/runner/hooks/evaluation.py
index 72a57be940..9447699151 100644
--- a/mmcv/runner/hooks/evaluation.py
+++ b/mmcv/runner/hooks/evaluation.py
@@ -46,12 +46,12 @@ class EvalHook(Hook):
 
     Notes:
         If new arguments are added for EvalHook, tools/test.py,
-        tools/eval_metric.py may be effected.
+        tools/eval_metric.py may be affected.
     """
 
-    # Since the key for determine greater an less is related to the downstream
-    # task, downstream repos may need to overwrite the following inner
-    # variable according to their tasks
+    # Since the key for determine greater or less is related to the downstream
+    # tasks, downstream repos may need to overwrite the following inner
+    # variable accordingly.
 
     rule_map = {'greater': lambda x, y: x > y, 'less': lambda x, y: x < y}
     init_value_map = {'greater': -inf, 'less': inf}
@@ -86,7 +86,8 @@ def __init__(self,
         self.by_epoch = by_epoch
 
         assert isinstance(save_best, str) or save_best is None, \
-            '``save_best`` should be a str or None'
+            '""save_best"" should be a str or None ' \
+            f'rather than {type(save_best)}'
         self.save_best = save_best
         self.eval_kwargs = eval_kwargs
         self.initial_flag = True
@@ -339,7 +340,6 @@ def __init__(self,
 
     def _do_evaluate(self, runner):
         """perform evaluation and save ckpt."""
-
         # Synchronization of BatchNorm's buffer (running_mean
         # and running_var) is not supported in the DDP of pytorch,
         # which may cause the inconsistent performance of models in