Merge develop to develop-idev

.github/workflows/run_tests_in_tox.yml

@@ -56,7 +56,6 @@ jobs:
           name: ${{ inputs.artifact-prefix }}-${{ inputs.toxenv-task }}-${{ inputs.toxenv-pyver }}-${{ inputs.toxenv-ptver }}
           path: |
             .tox/tests-${{ inputs.toxenv-task }}-${{ inputs.toxenv-pyver }}-${{ inputs.toxenv-ptver }}.csv
-            .tox/tests-reg_${{ inputs.task }}_*.csv
-            .tox/tests-reg_tiling_${{ inputs.task }}_*.csv
+            .tox/tests-reg_${{ inputs.task }}*.csv
         # Use always() to always run this step to publish test results when there are test failures
         if: ${{ inputs.upload-artifact && always() }}

.github/workflows/weekly.yml

@@ -14,31 +14,24 @@ jobs:
         include:
           - toxenv_task: "iseg"
             test_dir: "tests/regression/instance_segmentation/test_instance_segmentation.py"
-            runs_on: "['self-hosted', 'Linux', 'X64', 'dmount']"
             task: "instance_segmentation"
           - toxenv_task: "iseg_t"
             test_dir: "tests/regression/instance_segmentation/test_tiling_instance_segmentation.py"
-            runs_on: "['self-hosted', 'Linux', 'X64', 'dmount']"
             task: "instance_segmentation"
           - toxenv_task: "seg"
             test_dir: "tests/regression/semantic_segmentation"
-            runs_on: "['self-hosted', 'Linux', 'X64', 'dmount']"
             task: "segmentation"
           - toxenv_task: "det"
             test_dir: "tests/regression/detection"
-            runs_on: "['self-hosted', 'Linux', 'X64', 'dmount']"
             task: "detection"
           - toxenv_task: "ano"
             test_dir: "tests/regression/anomaly"
-            runs_on: "['self-hosted', 'Linux', 'X64', 'dmount']"
             task: "anomaly"
           - toxenv_task: "act"
             test_dir: "tests/regression/action"
-            runs_on: "['self-hosted', 'Linux', 'X64', 'dmount']"
             task: "action"
           - toxenv_task: "cls"
             test_dir: "tests/regression/classification"
-            runs_on: "['self-hosted', 'Linux', 'X64', 'dmount']"
             task: "classification"
     name: Regression-Test-py310-${{ matrix.toxenv_task }}
     uses: ./.github/workflows/run_tests_in_tox.yml
@@ -47,7 +40,7 @@ jobs:
       toxenv-pyver: "py310"
       toxenv-task: ${{ matrix.toxenv_task }}
       tests-dir: ${{ matrix.test_dir }}
-      runs-on: ${{ matrix.runs_on }}
+      runs-on: "['self-hosted', 'Linux', 'X64', 'dmount']"
       task: ${{ matrix.task }}
       timeout-minutes: 8640
       upload-artifact: true

.gitignore

@@ -18,6 +18,7 @@ results/
 build/
 dist/
 !src/otx/recipes/**
+src/otx/recipes/**/__pycache__
 *egg-info
 
 *.pth
@@ -45,3 +46,6 @@ src/**/*.so
 
 # Dataset made by unit-test
 tests/**/detcon_mask/*
+
+# sphinx-autosummary generated files
+docs/**/_autosummary/

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 
 ## \[unreleased\]
 
+### New features
+
+- Add zero-shot visual prompting (https://github.com/openvinotoolkit/training_extensions/pull/2616)
+
 ## \[v1.5.0\]
 
 ### New features
@@ -46,6 +50,7 @@ All notable changes to this project will be documented in this file.
 - Update ModelAPI configuration(<https://github.com/openvinotoolkit/training_extensions/pull/2564>)
 - Add Anomaly modelAPI changes (<https://github.com/openvinotoolkit/training_extensions/pull/2563>)
 - Update Image numpy access (<https://github.com/openvinotoolkit/training_extensions/pull/2586>)
+- Make max_num_detections configurable (<https://github.com/openvinotoolkit/training_extensions/pull/2647>)
 
 ### Bug fixes
 

docs/Makefile

@@ -23,3 +23,9 @@ html:
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+# Custom clean target that also removes autosummary generated files. Can
+# be removed when https://github.com/sphinx-doc/sphinx/issues/1999 is fixed.
+clean:
+	rm -rf "$(SOURCEDIR)/guide/reference/_autosummary"
+	$(SPHINXBUILD)  -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/source/conf.py

@@ -33,8 +33,23 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
+    "sphinx.ext.napoleon",  # Support for NumPy and Google style docstrings
     'sphinx.ext.autodoc',
     'sphinx_copybutton',
+    "sphinx.ext.autosummary",  # Create neat summary tables
+    "sphinx.ext.viewcode",  # Find the source files
+    "sphinx.ext.autosectionlabel",  # Refer sections its title
+    "sphinx.ext.intersphinx",  # Generate links to the documentation
+]
+
+source_suffix = {
+    ".rst": "restructuredtext",
+     ".md": "markdown",
+}
+
+suppress_warnings = [
+    "ref.python",
+    "autosectionlabel.*",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -45,7 +60,6 @@
 # This pattern also affects html_static_path and html_extra_path.
 exclude_patterns = []
 
-
 # -- Options for HTML output ------------------------------------------------- #
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
@@ -74,3 +88,25 @@
 html_css_files = [
     'css/custom.css',
 ]
+
+# -- Extension configuration -------------------------------------------------
+autodoc_docstring_signature = True
+autodoc_member_order = "bysource"
+intersphinx_mapping = {
+    "python": ("https://docs.python.org/3", None),
+    "numpy": ("https://numpy.org/doc/stable/", None),
+}
+autodoc_member_order = "groupwise"
+autodoc_default_options = {
+    "members": True,
+    "methods": True,
+    "special-members": "__call__",
+    "exclude-members": "_abc_impl",
+    "show-inheritance": True,
+}
+
+autoclass_content = "both"
+
+autosummary_generate = True     # Turn on sphinx.ext.autosummary
+autosummary_ignore_module_all = False   # Summary list in __all__ no others
+# autosummary_imported_members = True # document classes and functions imported in modules

docs/source/guide/explanation/additional_features/models_optimization.rst

@@ -1,17 +1,17 @@
 Models Optimization
 ===================
 
-OpenVINO™ Training Extensions provides two types of optimization algorithms: `Post-training Optimization Tool (POT) <https://docs.openvino.ai/latest/pot_introduction.html#doxid-pot-introduction>`_ and `Neural Network Compression Framework (NNCF) <https://github.com/openvinotoolkit/nncf>`_.
+OpenVINO™ Training Extensions provides two types of optimization algorithms: `Post-Training Quantization tool (PTQ) <https://github.com/openvinotoolkit/nncf#post-training-quantization>`_ and `Neural Network Compression Framework (NNCF) <https://github.com/openvinotoolkit/nncf>`_.
 
 *******************************
-Post-training Optimization Tool
+Post-Training Quantization Tool
 *******************************
 
-POT is designed to optimize the inference of models by applying post-training methods that do not require model retraining or fine-tuning. If you want to know more details about how POT works and to be more familiar with model optimization methods, please refer to `documentation <https://docs.openvino.ai/latest/pot_introduction.html#doxid-pot-introduction>`_.
+PTQ is designed to optimize the inference of models by applying post-training methods that do not require model retraining or fine-tuning. If you want to know more details about how PTQ works and to be more familiar with model optimization methods, please refer to `documentation <https://docs.openvino.ai/2023.2/ptq_introduction.html>`_.
 
-To run Post-training optimization it is required to convert the model to OpenVINO™ intermediate representation (IR) first. To perform fast and accurate quantization we use ``DefaultQuantization Algorithm`` for each task. Please, see the `DefaultQuantization Parameters <https://docs.openvino.ai/latest/pot_compression_algorithms_quantization_default_README.html#doxid-pot-compression-algorithms-quantization-default-r-e-a-d-m-e>`_ for further information about configuring the optimization.
+To run Post-training quantization it is required to convert the model to OpenVINO™ intermediate representation (IR) first. To perform fast and accurate quantization we use ``DefaultQuantization Algorithm`` for each task. Please, refer to the `Tune quantization Parameters <https://docs.openvino.ai/2023.2/basic_quantization_flow.html#tune-quantization-parameters>`_ for further information about configuring the optimization.
 
-POT parameters can be found and configured in ``template.yaml`` and ``configuration.yaml`` for each task. For Anomaly and Semantic Segmentation tasks, we have separate configuration files for POT, that can be found in the same directory with ``template.yaml``, for example for `PaDiM <https://github.com/openvinotoolkit/training_extensions/blob/develop/src/otx/algorithms/anomaly/configs/classification/padim/ptq_optimization_config.py>`_, `OCR-Lite-HRNe-18-mod2 <https://github.com/openvinotoolkit/training_extensions/blob/develop/src/otx/algorithms/segmentation/configs/ocr_lite_hrnet_18_mod2/ptq_optimization_config.py>`_ model.
+PTQ parameters can be found and configured in ``template.yaml`` and ``configuration.yaml`` for each task. For Anomaly and Semantic Segmentation tasks, we have separate configuration files for PTQ, that can be found in the same directory with ``template.yaml``, for example for `PaDiM <https://github.com/openvinotoolkit/training_extensions/blob/develop/src/otx/algorithms/anomaly/configs/classification/padim/ptq_optimization_config.py>`_, `OCR-Lite-HRNe-18-mod2 <https://github.com/openvinotoolkit/training_extensions/blob/develop/src/otx/algorithms/segmentation/configs/ocr_lite_hrnet_18_mod2/ptq_optimization_config.py>`_ model.
 
 ************************************
 Neural Network Compression Framework
@@ -25,9 +25,9 @@ You can refer to configuration files for default templates for each task accordi
 
 NNCF tends to provide better quality in terms of preserving accuracy as it uses training compression approaches.
 Compression results achievable with the NNCF can be found `here <https://github.com/openvinotoolkit/nncf#nncf-compressed-model-zoo>`_ .
-Meanwhile, the POT is faster but can degrade accuracy more than the training-enabled approach.
+Meanwhile, the PTQ is faster but can degrade accuracy more than the training-enabled approach.
 
 .. note::
     The main recommendation is to start with post-training compression and use NNCF compression during training if you are not satisfied with the results.
 
-Please, refer to our :doc:`dedicated tutorials <../../tutorials/base/how_to_train/index>` on how to optimize your model using POT or NNCF.
+Please, refer to our :doc:`dedicated tutorials <../../tutorials/base/how_to_train/index>` on how to optimize your model using PTQ or NNCF.

docs/source/guide/explanation/algorithms/action/action_classification.rst

@@ -1,5 +1,5 @@
 Action Classification
-==================
+=====================
 
 Action classification is a problem of identifying the action that is being performed in a video. The input to the algorithm is a sequence of video frames, and the output is a label indicating the action that is being performed.
 

docs/source/guide/explanation/algorithms/anomaly/index.rst

@@ -143,7 +143,7 @@ Since STFPM trains the student network, we use the following parameters for its
 - ``Aditional Techniques``:
    - ``Early Stopping``: Early stopping is used to stop the training process when the validation loss stops improving. The default value of the early stopping patience is ``10``.
 
-For more information on STFPM's training. We invite you to read Anomalib's `STFPM documentation<https://openvinotoolkit.github.io/anomalib/reference_guide/algorithms/stfpm.html>`_.
+For more information on STFPM's training. We invite you to read Anomalib's `STFPM documentation <https://anomalib.readthedocs.io/en/latest/reference_guide/algorithms/stfpm.htm>`_.
 
 Reconstruction-based Models
 ---------------------------

docs/source/guide/explanation/algorithms/classification/multi_class_classification.rst

@@ -100,7 +100,7 @@ In the table below the top-1 accuracy on some academic datasets using our :ref:`
 +-----------------------+-----------------+-----------+-----------+-----------+
 | EfficientNet-V2-S     | 96.13           | 90.36     | 97.68     | 86.74     |
 +-----------------------+-----------------+-----------+-----------+-----------+
-*These datasets were splitted with auto-split (80% train, 20% test).
+\* These datasets were splitted with auto-split (80% train, 20% test).
 
 ************************
 Semi-supervised Learning
@@ -145,7 +145,6 @@ In the table below the top-1 accuracy on some academic datasets using our pipeli
 |  EfficientNet-V2-S    |  36.03  |  39.66  | 16.81 |  20.28  | 65.99  |  69.61  |
 +-----------------------+---------+---------+-------+---------+--------+---------+
 
-|
 
 - 10 labeled images per class including unlabeled dataset for Semi-SL
 

docs/source/guide/explanation/algorithms/classification/multi_label_classification.rst

@@ -28,6 +28,7 @@ Specifically, this format should be converted in our `internal representation <h
 To convert the COCO data format to our internal one, run this script in similar way:
 
 .. code-block::
+
     python convert_coco_to_multilabel.py --ann_file_path <path to .json COCO annotations> --data_root_dir <path to images folder> --output <output path to save annotations>
 
 .. note::

docs/source/guide/explanation/algorithms/index.rst

@@ -11,7 +11,9 @@ To this end, we support:
 - **Supervised training**. This is the most common approach for computer vision tasks such as object detection and image classification. Supervised learning involves training a model on a labeled dataset of images. The model learns to associate specific features in the images with the corresponding labels.
 
 - **Incremental learning**. This learning approach lets the model train on new data as it becomes available, rather than retraining the entire model on the whole dataset every time new data is added. OpenVINO™ Training Extensions supports also the class incremental approach for all tasks. In this approach, the model is first trained on a set of classes, and then incrementally updated with new classes of data, while keeping the previously learned classes' knowledge. The class incremental approach is particularly useful in situations where the number of classes is not fixed and new classes may be added over time.
+
 .. _semi_sl_explanation:
+
 - **Semi-supervised learning**.  This is a type of machine learning in which the model is trained on a dataset that contains a combination of labeled and unlabeled examples. The labeled examples are used to train the model, while the unlabeled examples are used to improve the model's performance by providing additional information about the underlying distribution of the data. This approach is often used when there is a limited amount of labeled data available, but a large amount of unlabeled data. This can make it more cost-effective and efficient to train models compared to traditional supervised learning, where the model is trained only on labeled data.
 
 - **Self-supervised learning**. This is a type of machine learning where the model is trained on a dataset that contains only unlabeled examples. The model is trained to learn useful representations of the data by solving a task that can be inferred from the input itself, without human-provided labels. The objective is to learn good representations of the input data that can then be used for downstream tasks such as classification, detection, generation or clustering.

docs/source/guide/explanation/algorithms/object_detection/object_detection.rst

@@ -92,6 +92,7 @@ We support the following ready-to-use model templates:
 Above table can be found using the following command
 
 .. code-block::
+
     $ otx find --task detection
 
 `MobileNetV2-ATSS <https://arxiv.org/abs/1912.02424>`_ is a good medium-range model that works well and fast in most cases.
@@ -147,6 +148,7 @@ Please, refer to the :doc:`tutorial <../../../tutorials/advanced/backbones>` how
 To see which public backbones are available for the task, the following command can be executed:
 
 .. code-block::
+
     $ otx find --backbone {torchvision, pytorchcv, mmcls, omz.mmcls}
 
 In the table below the test mAP on some academic datasets using our :ref:`supervised pipeline <od_supervised_pipeline>` is presented.

docs/source/guide/explanation/algorithms/segmentation/instance_segmentation.rst

@@ -61,11 +61,11 @@ We support the following ready-to-use model templates:
 +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------+---------------------+-----------------+
 | Template ID                                                                                                                                                                                                                                | Name                       | Complexity (GFLOPs) | Model size (MB) |
 +============================================================================================================================================================================================================================================+============================+=====================+=================+
-| `Custom_Counting_Instance_Segmentation_MaskRCNN_EfficientNetB2B <https://github.com/openvinotoolkit/training_extensions/blob/develop/src/otx/algorithms/detection/configs/instance_segmentation/efficientnetb2b_maskrcnn/template.yaml>`_      | MaskRCNN-EfficientNetB2B   | 68.48           | 13.27           |
+| `Custom_Counting_Instance_Segmentation_MaskRCNN_EfficientNetB2B <https://github.com/openvinotoolkit/training_extensions/blob/develop/src/otx/algorithms/detection/configs/instance_segmentation/efficientnetb2b_maskrcnn/template.yaml>`_  | MaskRCNN-EfficientNetB2B   | 68.48               | 13.27           |
 +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------+---------------------+-----------------+
-| `Custom_Counting_Instance_Segmentation_MaskRCNN_ResNet50 <https://github.com/openvinotoolkit/training_extensions/blob/develop/src/otx/algorithms/detection/configs/instance_segmentation/resnet50_maskrcnn/template.yaml>`_                    | MaskRCNN-ResNet50          | 533.80          | 177.90          |
+| `Custom_Counting_Instance_Segmentation_MaskRCNN_ResNet50 <https://github.com/openvinotoolkit/training_extensions/blob/develop/src/otx/algorithms/detection/configs/instance_segmentation/resnet50_maskrcnn/template.yaml>`_                | MaskRCNN-ResNet50          | 533.80              | 177.90          |
 +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------+---------------------+-----------------+
-| `Custom_Counting_Instance_Segmentation_MaskRCNN_ConvNeXt <https://github.com/openvinotoolkit/training_extensions/blob/develop/src/otx/algorithms/detection/configs/instance_segmentation/convnext_maskrcnn/template.yaml>`_                    | MaskRCNN-ConvNeXt          | 266.78          | 192.4          |
+| `Custom_Counting_Instance_Segmentation_MaskRCNN_ConvNeXt <https://github.com/openvinotoolkit/training_extensions/blob/develop/src/otx/algorithms/detection/configs/instance_segmentation/convnext_maskrcnn/template.yaml>`_                | MaskRCNN-ConvNeXt          | 266.78              | 192.4           |
 +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------+---------------------+-----------------+
 
 MaskRCNN-ResNet50 utilizes the `ResNet-50 <https://arxiv.org/abs/1512.03385>`_ architecture as the backbone network for extracting image features. This choice of backbone network results in a higher number of parameters and FLOPs, which consequently requires more training time. However, the model offers superior performance in terms of accuracy.