diff --git a/.mypy.ini b/.mypy.ini new file mode 100644 index 00000000000..532737e2c07 --- /dev/null +++ b/.mypy.ini @@ -0,0 +1,51 @@ +[mypy] +python_version = 3.8 +ignore_missing_imports = True +show_error_codes = True +check_untyped_defs = True +strict = True + +[mypy-mmdet.core.*] +ignore_errors = True + +[mypy-mmdet.datasets.*] +ignore_errors = True + +[mypy-mmdet.integration.*] +ignore_errors = True + +[mypy-mmdet.models.*] +ignore_errors = True + +[mypy-mmdet.ops.*] +ignore_errors = True + +[mypy-mmdet.parallel.*] +ignore_errors = True + +[mypy-mmdet.utils.*] +ignore_errors = True + +[mypy-mmdet] +ignore_errors = True + +[mypy-mmdet.version] +ignore_errors = True + +[mypy-mmdet.apis] +ignore_errors = True + +[mypy-mmdet.apis.train] +ignore_errors = True + +[mypy-mmdet.apis.test] +ignore_errors = True + +[mypy-mmdet.apis.export] +ignore_errors = True + +[mypy-mmdet.apis.inference] +ignore_errors = True + +[mypy-mmdet.apis.fake_input] +ignore_errors = True diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 35566b35337..5beda0b2ec2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,40 +1,47 @@ repos: - - repo: https://gitlab.com/pycqa/flake8.git - rev: 3.8.3 + # - repo: https://gitlab.com/pycqa/flake8.git + # rev: 3.8.3 + # hooks: + # - id: flake8 + # - repo: https://github.com/asottile/seed-isort-config + # rev: v2.2.0 + # hooks: + # - id: seed-isort-config + # - repo: https://github.com/timothycrosley/isort + # rev: 4.3.21 + # hooks: + # - id: isort + # - repo: https://github.com/pre-commit/mirrors-yapf + # rev: v0.30.0 + # hooks: + # - id: yapf + # - repo: https://github.com/pre-commit/pre-commit-hooks + # rev: v3.1.0 + # hooks: + # - id: trailing-whitespace + # - id: check-yaml + # - id: end-of-file-fixer + # - id: requirements-txt-fixer + # - id: double-quote-string-fixer + # - id: check-merge-conflict + # - id: fix-encoding-pragma + # args: ["--remove"] + # - id: mixed-line-ending + # args: ["--fix=lf"] + # - repo: https://github.com/jumanjihouse/pre-commit-hooks + # rev: 2.1.4 + # hooks: + # - id: markdownlint + # args: ["-r", "~MD002,~MD013,~MD024,~MD029,~MD033,~MD034,~MD036"] + # - repo: https://github.com/myint/docformatter + # rev: v1.3.1 + # hooks: + # - id: docformatter + # args: ["--in-place", "--wrap-descriptions", "79"] + - repo: https://github.com/pre-commit/mirrors-mypy + rev: 'v0.812' hooks: - - id: flake8 - - repo: https://github.com/asottile/seed-isort-config - rev: v2.2.0 - hooks: - - id: seed-isort-config - - repo: https://github.com/timothycrosley/isort - rev: 4.3.21 - hooks: - - id: isort - - repo: https://github.com/pre-commit/mirrors-yapf - rev: v0.30.0 - hooks: - - id: yapf - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.1.0 - hooks: - - id: trailing-whitespace - - id: check-yaml - - id: end-of-file-fixer - - id: requirements-txt-fixer - - id: double-quote-string-fixer - - id: check-merge-conflict - - id: fix-encoding-pragma - args: ["--remove"] - - id: mixed-line-ending - args: ["--fix=lf"] - - repo: https://github.com/jumanjihouse/pre-commit-hooks - rev: 2.1.4 - hooks: - - id: markdownlint - args: ["-r", "~MD002,~MD013,~MD024,~MD029,~MD033,~MD034,~MD036"] - - repo: https://github.com/myint/docformatter - rev: v1.3.1 - hooks: - - id: docformatter - args: ["--in-place", "--wrap-descriptions", "79"] + - id: mypy + args: ["--config-file=.mypy.ini"] + additional_dependencies: [numpy==1.19.5, types-PyYAML, attrs==21.2.*, types-requests, types-Deprecated, types-docutils, types_futures, types-python-dateutil] + exclude: "ci|configs|data|demo|docker|docs|experiments|resources|tools|.dev_scripts|tests|setup.py" diff --git a/mmdet/apis/ote/apis/detection/__init__.py b/mmdet/apis/ote/apis/detection/__init__.py index 03c0dce1f69..4848f51f08b 100644 --- a/mmdet/apis/ote/apis/detection/__init__.py +++ b/mmdet/apis/ote/apis/detection/__init__.py @@ -24,19 +24,19 @@ from .train_task import OTEDetectionTrainingTask __all__ = [ - config_from_string, - config_to_string, - generate_label_schema, - get_task_class, - load_template, - OpenVINODetectionTask, - OTEDetectionConfig, - OTEDetectionInferenceTask, - OTEDetectionNNCFTask, - OTEDetectionTrainingTask, - patch_config, - prepare_for_testing, - prepare_for_training, - save_config_to_file, - set_hyperparams, - ] + 'config_from_string', + 'config_to_string', + 'generate_label_schema', + 'get_task_class', + 'load_template', + 'OpenVINODetectionTask', + 'OTEDetectionConfig', + 'OTEDetectionInferenceTask', + 'OTEDetectionNNCFTask', + 'OTEDetectionTrainingTask', + 'patch_config', + 'prepare_for_testing', + 'prepare_for_training', + 'save_config_to_file', + 'set_hyperparams', +] diff --git a/mmdet/apis/ote/apis/detection/inference_task.py b/mmdet/apis/ote/apis/detection/inference_task.py index 248d1100c93..e76423c7a29 100644 --- a/mmdet/apis/ote/apis/detection/inference_task.py +++ b/mmdet/apis/ote/apis/detection/inference_task.py @@ -221,7 +221,7 @@ def hook(module, input, output): @staticmethod def _infer_detector(model: torch.nn.Module, config: Config, dataset: DatasetEntity, dump_features: bool = False, - eval: Optional[bool] = False, metric_name: Optional[str] = 'mAP') -> Tuple[List, float]: + eval: bool = False, metric_name: str = 'mAP') -> Tuple[List, float]: model.eval() test_config = prepare_for_testing(config, dataset) mm_val_dataset = build_dataset(test_config.data.test) @@ -256,7 +256,7 @@ def dummy_dump_features_hook(mod, inp, out): result = eval_model(return_loss=False, rescale=True, **data) eval_predictions.extend(result) - metric = None + metric = 0.0 if eval: metric = mm_val_dataset.evaluate(eval_predictions, metric=metric_name)[metric_name] diff --git a/mmdet/apis/ote/apis/detection/nncf_task.py b/mmdet/apis/ote/apis/detection/nncf_task.py index d718d6d277d..1487ed569f7 100644 --- a/mmdet/apis/ote/apis/detection/nncf_task.py +++ b/mmdet/apis/ote/apis/detection/nncf_task.py @@ -17,7 +17,7 @@ import logging import os from collections import defaultdict -from typing import Optional +from typing import DefaultDict, Optional import torch from ote_sdk.configuration import cfg_helper @@ -189,7 +189,7 @@ def optimize( else: update_progress_callback = default_progress_callback time_monitor = TrainingProgressCallback(update_progress_callback) - learning_curves = defaultdict(OTELoggerHook.Curve) + learning_curves: DefaultDict[str, OTELoggerHook.Curve] = defaultdict(OTELoggerHook.Curve) training_config = prepare_for_training(config, train_dataset, val_dataset, time_monitor, learning_curves) mm_train_dataset = build_dataset(training_config.data.train) @@ -235,7 +235,7 @@ def save_model(self, output_model: ModelEntity): hyperparams_str = ids_to_strings(cfg_helper.convert(hyperparams, dict, enum_to_str=True)) labels = {label.name: label.color.rgb_tuple for label in self._labels} modelinfo = { - 'compression_state': self._compression_ctrl.get_compression_state(), + 'compression_state': self._compression_ctrl.get_compression_state(), # type: ignore # FIXME. 'meta': { 'config': self._config, 'nncf_enable_compression': True, diff --git a/mmdet/apis/ote/apis/detection/openvino_task.py b/mmdet/apis/ote/apis/detection/openvino_task.py index b20821ddd19..d3cf530d77e 100644 --- a/mmdet/apis/ote/apis/detection/openvino_task.py +++ b/mmdet/apis/ote/apis/detection/openvino_task.py @@ -25,6 +25,7 @@ from compression.graph import load_model, save_model from compression.graph.model_utils import compress_model_weights, get_nodes_by_type from compression.pipeline.initializer import create_pipeline +from openvino.inference_engine import ExecutableNetwork from ote_sdk.entities.annotation import Annotation, AnnotationSceneEntity, AnnotationSceneKind from ote_sdk.entities.datasets import DatasetEntity from ote_sdk.entities.inference_parameters import InferenceParameters, default_progress_callback @@ -52,7 +53,7 @@ logger = logging.getLogger(__name__) -def get_output(net, outputs, name): +def get_output(net: ExecutableNetwork, outputs: Dict[str, np.ndarray], name: str) -> np.ndarray: try: key = net.get_ov_name_for_tensor(name) assert key in outputs, f'"{key}" is not a valid output identifier' @@ -63,7 +64,8 @@ def get_output(net, outputs, name): return outputs[key] -def extract_detections(output, net, input_size): +def extract_detections(output: Dict[str, np.ndarray], net: ExecutableNetwork, + input_size: Tuple[int, int]) -> Dict[str, np.ndarray]: if 'detection_out' in output: detection_out = output['detection_out'] output['labels'] = detection_out[0, 0, :, 1].astype(np.int32) @@ -114,7 +116,7 @@ def __init__( self.confidence_threshold = confidence_threshold @staticmethod - def resize_image(image: np.ndarray, size: Tuple[int], keep_aspect_ratio: bool = False) -> np.ndarray: + def resize_image(image: np.ndarray, size: Tuple[int, int], keep_aspect_ratio: bool = False) -> np.ndarray: if not keep_aspect_ratio: resized_frame = cv2.resize(image, size) else: diff --git a/mmdet/apis/ote/apis/detection/train_task.py b/mmdet/apis/ote/apis/detection/train_task.py index 686fb59ac66..8c94dedc1cd 100644 --- a/mmdet/apis/ote/apis/detection/train_task.py +++ b/mmdet/apis/ote/apis/detection/train_task.py @@ -18,7 +18,7 @@ import os from collections import defaultdict from glob import glob -from typing import List, Optional +from typing import DefaultDict, Dict, List, Optional import torch from ote_sdk.configuration import cfg_helper @@ -45,7 +45,7 @@ class OTEDetectionTrainingTask(OTEDetectionInferenceTask, ITrainingTask): - def _generate_training_metrics(self, learning_curves, map) -> Optional[List[MetricsGroup]]: + def _generate_training_metrics(self, learning_curves: Dict[str, OTELoggerHook.Curve], map: float) -> List[MetricsGroup]: """ Parses the mmdetection logs to get metrics from the latest training run @@ -97,7 +97,7 @@ def train(self, dataset: DatasetEntity, output_model: ModelEntity, train_paramet if train_parameters is not None: update_progress_callback = train_parameters.update_progress time_monitor = TrainingProgressCallback(update_progress_callback) - learning_curves = defaultdict(OTELoggerHook.Curve) + learning_curves: DefaultDict[str, OTELoggerHook.Curve] = defaultdict(OTELoggerHook.Curve) training_config = prepare_for_training(config, train_dataset, val_dataset, time_monitor, learning_curves) self._training_work_dir = training_config.work_dir mm_train_dataset = build_dataset(training_config.data.train) diff --git a/mmdet/apis/ote/extension/datasets/__init__.py b/mmdet/apis/ote/extension/datasets/__init__.py index 43526acec85..0785abed0da 100644 --- a/mmdet/apis/ote/extension/datasets/__init__.py +++ b/mmdet/apis/ote/extension/datasets/__init__.py @@ -14,4 +14,4 @@ from .mmdataset import OTEDataset, get_annotation_mmdet_format -__all__ = [OTEDataset, get_annotation_mmdet_format] +__all__ = ['OTEDataset', 'get_annotation_mmdet_format'] diff --git a/mmdet/apis/ote/extension/datasets/mmdataset.py b/mmdet/apis/ote/extension/datasets/mmdataset.py index cdbb6921328..3b878256ef2 100644 --- a/mmdet/apis/ote/extension/datasets/mmdataset.py +++ b/mmdet/apis/ote/extension/datasets/mmdataset.py @@ -13,7 +13,7 @@ # and limitations under the License. from copy import deepcopy -from typing import List +from typing import List, Sequence, Optional import numpy as np from ote_sdk.entities.dataset_item import DatasetItemEntity @@ -88,7 +88,7 @@ class _DataInfoProxy: forwards data access operations to ote_dataset and converts the dataset items to the view convenient for mmdetection. """ - def __init__(self, ote_dataset, classes): + def __init__(self, ote_dataset: DatasetEntity, classes: Optional[Sequence[str]]): self.ote_dataset = ote_dataset self.CLASSES = classes @@ -113,7 +113,7 @@ def __getitem__(self, index): return data_info def __init__(self, ote_dataset: DatasetEntity, pipeline, classes=None, test_mode: bool = False): - self.ote_dataset = ote_dataset + self.ote_dataset: DatasetEntity = ote_dataset self.test_mode = test_mode self.CLASSES = self.get_classes(classes) diff --git a/mmdet/apis/ote/extension/utils/__init__.py b/mmdet/apis/ote/extension/utils/__init__.py index 9cc35e5287d..eb7156da9b9 100644 --- a/mmdet/apis/ote/extension/utils/__init__.py +++ b/mmdet/apis/ote/extension/utils/__init__.py @@ -17,6 +17,6 @@ from .pipelines import LoadImageFromOTEDataset, LoadAnnotationFromOTEDataset from .runner import EpochRunnerWithCancel -__all__ = [CancelTrainingHook, FixedMomentumUpdaterHook, LoadImageFromOTEDataset, EpochRunnerWithCancel, - LoadAnnotationFromOTEDataset, OTELoggerHook, OTEProgressHook, EarlyStoppingHook, - ReduceLROnPlateauLrUpdaterHook] +__all__ = ['CancelTrainingHook', 'FixedMomentumUpdaterHook', 'LoadImageFromOTEDataset', 'EpochRunnerWithCancel', + 'LoadAnnotationFromOTEDataset', 'OTELoggerHook', 'OTEProgressHook', 'EarlyStoppingHook', + 'ReduceLROnPlateauLrUpdaterHook'] diff --git a/tests/conftest.py b/tests/conftest.py index e8207833e78..9a02c64cf1b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,7 +10,7 @@ _e2e_pytest_addoption = None pass -import config +import e2e_config def pytest_addoption(parser): if _e2e_pytest_addoption: diff --git a/tests/config.py b/tests/e2e_config.py similarity index 100% rename from tests/config.py rename to tests/e2e_config.py diff --git a/tests/test_ote_training.py b/tests/test_ote_training.py index 7e058ba2129..e95ce00a3af 100644 --- a/tests/test_ote_training.py +++ b/tests/test_ote_training.py @@ -21,7 +21,7 @@ from collections import namedtuple, OrderedDict from copy import deepcopy from pprint import pformat -from typing import List, Optional, Union +from typing import List, Optional, Tuple, Union import pytest import yaml @@ -34,10 +34,8 @@ from ote_sdk.entities.model import ( ModelEntity, ModelFormat, - ModelPrecision, ModelStatus, ModelOptimizationType, - OptimizationMethod, ) from ote_sdk.entities.model_template import parse_model_template, TargetDevice from ote_sdk.entities.optimization_parameters import OptimizationParameters @@ -53,7 +51,7 @@ logger = logging.getLogger(__name__) -def DATASET_PARAMETERS_FIELDS(): +def DATASET_PARAMETERS_FIELDS() -> Tuple[str, ...]: return ('annotations_train', 'images_train_dir', 'annotations_val', @@ -63,7 +61,7 @@ def DATASET_PARAMETERS_FIELDS(): ) ROOT_PATH_KEY = '_root_path' -DatasetParameters = namedtuple('DatasetParameters', DATASET_PARAMETERS_FIELDS()) +DatasetParameters = namedtuple('DatasetParameters', DATASET_PARAMETERS_FIELDS()) # type: ignore @pytest.fixture def dataset_definitions_fx(request): @@ -659,7 +657,7 @@ class OTETestStage: time the stage is called the exception is re-raised. """ def __init__(self, action: BaseOTETestAction, - depends_stages: Optional[List['OTETestStage']]=None): + depends_stages: Optional[List['OTETestStage']] = None): self.was_processed = False self.stored_exception = None self.action = action