Merge branch 'releases/v0.2.0-sc1.1' of https://github.com/openvinoto…

…olkit/training_extensions into ad/mmseg_mmdet_updates

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "external/mmsegmentation"]
 	path = external/mmsegmentation/submodule
 	url = ../../openvinotoolkit/mmsegmentation
+[submodule "external/model-preparation-algorithm/submodule"]
+	path = external/model-preparation-algorithm/submodule
+	url = https://github.com/openvinotoolkit/model_preparation_algorithm

CHANGELOG.md

@@ -2,10 +2,29 @@
 
 All notable changes to this project will be documented in this file.
 
+## \[v0.2.0\]
+### Added
+* Model Preparation Algorithm (MPA), a newly introduced OTE Algorithm backend for advanced transfer learning
+  * Class-Incremental Learning support for OTE models
+    * Image Classification
+    * Object Detection
+    * Semantic Segmentation
+* Object counting & Rotated object detection are added to Object Detection backend
+* Increased support for NNCF / FP16 / HPO
+* Ignored label support
+* Stop training on NaN losses
+
+
+### Changed
+* Major refactoring
+  * Tasks & model templates had been moved to OTE repo from each OTE Algorithm backend
+
+
 ## \[v0.1.1\]
 ### Fixed
 * Some minor issues
 
+
 ## \[v0.1.0\]
 ### Added
 * OTE SDK, defines an interface which can be used by OTE CLI to access OTE Algorithms.
@@ -15,3 +34,4 @@ All notable changes to this project will be documented in this file.
   * Image Classification
   * Object Detection
   * Semantic Segmentation
+

external/README.md

@@ -24,16 +24,23 @@ ote_anomaly_segmentation_stfpm | STFPM | 5.6 | 21.1 | anomaly/anomaly_segmentati
 ## Image Classification
 ID | Name | Complexity (GFlops) | Model size (MB) | Path
 ------- | ------- | ------- | ------- | -------
+ClassIncremental_Image_Classification_MobileNet-V3-small | MobileNet-V3-small-ClsIncr | 0.12 | 1.56 | model-preparation-algorithm/configs/classification/mobilenet_v3_small_cls_incr/template.yaml
+ClassIncremental_Image_Classification_MobileNet-V3-large-0.75x | MobileNet-V3-large-0.75x-ClsIncr | 0.32 | 2.76 | model-preparation-algorithm/configs/classification/mobilenet_v3_large_075_cls_incr/template.yaml
+ClassIncremental_Image_Classification_MobileNet-V3-large-1x | MobileNet-V3-large-1x-ClsIncr | 0.44 | 4.29 | model-preparation-algorithm/configs/classification/mobilenet_v3_large_1_cls_incr/template.yaml
 Custom_Image_Classification_MobileNet-V3-large-1x | MobileNet-V3-large-1x | 0.44 | 4.29 | deep-object-reid/configs/ote_custom_classification/mobilenet_v3_large_1/template.yaml
+ClassIncremental_Image_Classification_EfficinetNet-B0 | EfficientNet-B0-ClsIncr | 0.81 | 4.09 | model-preparation-algorithm/configs/classification/efficientnet_b0_cls_incr/template.yaml
 Custom_Image_Classification_EfficinetNet-B0 | EfficientNet-B0 | 0.81 | 4.09 | deep-object-reid/configs/ote_custom_classification/efficientnet_b0/template.yaml
+ClassIncremental_Image_Classification_EfficinetNet-V2-S | EfficientNet-V2-S-ClsIncr | 5.76 | 20.23 | model-preparation-algorithm/configs/classification/efficientnet_v2_s_cls_incr/template.yaml
 Custom_Image_Classification_EfficientNet-V2-S | EfficientNet-V2-S | 5.76 | 20.23 | deep-object-reid/configs/ote_custom_classification/efficientnet_v2_s/template.yaml
 
 ## Object Detection
 ID | Name | Complexity (GFlops) | Model size (MB) | Path
 ------- | ------- | ------- | ------- | -------
 Custom_Object_Detection_YOLOX | YOLOX | 6.5 | 20.4 | mmdetection/configs/custom-object-detection/cspdarknet_YOLOX/template.yaml
 Custom_Object_Detection_Gen3_SSD | SSD | 9.4 | 7.6 | mmdetection/configs/custom-object-detection/gen3_mobilenetV2_SSD/template.yaml
+ClassIncremental_Object_Detection_Gen3_ATSS | ATSS-ClsIncr | 20.6 | 9.1 | model-preparation-algorithm/configs/detection/mobilenetv2_atss_cls_incr/template.yaml
 Custom_Object_Detection_Gen3_ATSS | ATSS | 20.6 | 9.1 | mmdetection/configs/custom-object-detection/gen3_mobilenetV2_ATSS/template.yaml
+ClassIncremental_Object_Detection_Gen3_VFNet | VFNet-ClsIncr | 457.4 | 126.0 | model-preparation-algorithm/configs/detection/resnet50_vfnet_cls_incr/template.yaml
 
 ## Object Counting
 ID | Name | Complexity (GFlops) | Model size (MB) | Path
@@ -51,6 +58,7 @@ Custom_Rotated_Detection_via_Instance_Segmentation_MaskRCNN_ResNet50 | MaskRCNN-
 ID | Name | Complexity (GFlops) | Model size (MB) | Path
 ------- | ------- | ------- | ------- | -------
 Custom_Semantic_Segmentation_Lite-HRNet-s-mod2_OCR | Lite-HRNet-s-mod2 OCR | 1.82 | 3.5 | mmsegmentation/configs/custom-sematic-segmentation/ocr-lite-hrnet-s-mod2/template.yaml
+ClassIncremental_Semantic_Segmentation_Lite-HRNet-18_OCR | Lite-HRNet-18 OCR-ClsIncr | 3.45 | 4.5 | model-preparation-algorithm/configs/segmentation/ocr-lite-hrnet-18-cls-incr/template.yaml
 Custom_Semantic_Segmentation_Lite-HRNet-18_OCR | Lite-HRNet-18 OCR | 3.45 | 4.5 | mmsegmentation/configs/custom-sematic-segmentation/ocr-lite-hrnet-18/template.yaml
 Custom_Semantic_Segmentation_Lite-HRNet-18-mod2_OCR | Lite-HRNet-18-mod2 OCR | 3.63 | 4.8 | mmsegmentation/configs/custom-sematic-segmentation/ocr-lite-hrnet-18-mod2/template.yaml
 Custom_Semantic_Segmentation_Lite-HRNet-x-mod3_OCR | Lite-HRNet-x-mod3 OCR | 13.97 | 6.4 | mmsegmentation/configs/custom-sematic-segmentation/ocr-lite-hrnet-x-mod3/template.yaml

external/anomaly/constraints.txt

@@ -16,3 +16,4 @@ pytorch-lightning==1.5.9
 requests==2.26.0
 scikit-image==0.17.2
 scikit-learn==0.24.2
+torchmetrics==0.6.0

external/anomaly/ote_anomalib/data/create_mvtec_ad_json_annotations.py

@@ -87,7 +87,7 @@ def create_bboxes_from_mask(mask_path: str) -> List[List[float]]:
     return bboxes
 
 
-def create_polygons_from_mask(mask_path: str) -> List[List[float]]:
+def create_polygons_from_mask(mask_path: str) -> List[List[List[float]]]:
     """Create polygons from binary mask.
 
     Args:
@@ -99,8 +99,8 @@ def create_polygons_from_mask(mask_path: str) -> List[List[float]]:
     mask = cv2.imread(mask_path, cv2.IMREAD_GRAYSCALE)
     height, width = mask.shape
 
-    polygons = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)[0][0]
-    polygons = [[x / width, y / height] for polygon in polygons for (x, y) in polygon]
+    polygons = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)[0]
+    polygons = [[[point[0][0] / width, point[0][1] / height] for point in polygon] for polygon in polygons]
 
     return polygons
 

external/anomaly/ote_anomalib/data/data.py

@@ -206,7 +206,7 @@ def val_dataloader(self) -> Union[DataLoader, List[DataLoader]]:
         logger.info(f"Local annotations: {len(local_dataset)}")
         if contains_anomalous_images(local_dataset):
             logger.info("Dataset contains polygon annotations. Passing masks to anomalib.")
-            dataset = OTEAnomalyDataset(self.config, local_dataset, TaskType.ANOMALY_SEGMENTATION)
+            dataset = OTEAnomalyDataset(self.config, local_dataset, self.task_type)
         else:
             logger.info("Dataset does not contain polygon annotations. Not passing masks to anomalib.")
             dataset = OTEAnomalyDataset(self.config, global_dataset, TaskType.ANOMALY_CLASSIFICATION)

ote_cli/ote_cli/datasets/anomaly/dataset.py → external/anomaly/ote_anomalib/data/dataset.py

@@ -55,9 +55,7 @@ def __init__(
                 and dataset used for testing. Defaults to None.
         """
         items: List[DatasetItemEntity] = []
-        self.normal_label = LabelEntity(
-            id=ID(0), name="Normal", domain=Domain.ANOMALY_CLASSIFICATION
-        )
+        self.normal_label = LabelEntity(id=ID(0), name="Normal", domain=Domain.ANOMALY_CLASSIFICATION)
         self.abnormal_label = LabelEntity(
             id=ID(1),
             name="Anomalous",
@@ -101,9 +99,7 @@ def __init__(
         super().__init__(items=items)
 
     @abstractmethod
-    def get_dataset_items(
-        self, ann_file_path: Path, data_root_dir: Path, subset: Subset
-    ) -> List[DatasetItemEntity]:
+    def get_dataset_items(self, ann_file_path: Path, data_root_dir: Path, subset: Subset) -> List[DatasetItemEntity]:
         """To be implemented ib subclasses."""
         raise NotImplementedError
 
@@ -124,9 +120,7 @@ class AnomalyClassificationDataset(BaseAnomalyDataset):
     >>> testing_dataset = AnomalyClassificationDataset(test_subset=test_subset)
     """
 
-    def get_dataset_items(
-        self, ann_file_path: Path, data_root_dir: Path, subset: Subset
-    ) -> List[DatasetItemEntity]:
+    def get_dataset_items(self, ann_file_path: Path, data_root_dir: Path, subset: Subset) -> List[DatasetItemEntity]:
         """Loads dataset based on the image path in annotation file.
 
         Args:
@@ -148,19 +142,13 @@ def get_dataset_items(
             image = Image(file_path=str(data_root_dir / sample.image_path))
             # Create annotation
             shape = Rectangle.generate_full_box()
-            label: LabelEntity = (
-                self.normal_label if sample.label == "good" else self.abnormal_label
-            )
+            label: LabelEntity = self.normal_label if sample.label == "good" else self.abnormal_label
             labels = [ScoredLabel(label, probability=1.0)]
             annotations = [Annotation(shape=shape, labels=labels)]
-            annotation_scene = AnnotationSceneEntity(
-                annotations=annotations, kind=AnnotationSceneKind.ANNOTATION
-            )
+            annotation_scene = AnnotationSceneEntity(annotations=annotations, kind=AnnotationSceneKind.ANNOTATION)
 
             # Create dataset item
-            dataset_item = DatasetItemEntity(
-                media=image, annotation_scene=annotation_scene, subset=subset
-            )
+            dataset_item = DatasetItemEntity(media=image, annotation_scene=annotation_scene, subset=subset)
             # Add to dataset items
             dataset_items.append(dataset_item)
 
@@ -184,9 +172,7 @@ class AnomalySegmentationDataset(BaseAnomalyDataset):
 
     """
 
-    def get_dataset_items(
-        self, ann_file_path: Path, data_root_dir: Path, subset: Subset
-    ) -> List[DatasetItemEntity]:
+    def get_dataset_items(self, ann_file_path: Path, data_root_dir: Path, subset: Subset) -> List[DatasetItemEntity]:
         """Loads dataset based on the image path in annotation file.
 
         Args:
@@ -207,9 +193,7 @@ def get_dataset_items(
             # convert path to str as PosixPath is not supported by Image
             image = Image(file_path=str(data_root_dir / sample.image_path))
             # Create annotation
-            label: LabelEntity = (
-                self.normal_label if sample.label == "good" else self.abnormal_label
-            )
+            label: LabelEntity = self.normal_label if sample.label == "good" else self.abnormal_label
             annotations = [
                 Annotation(
                     Rectangle.generate_full_box(),
@@ -237,16 +221,10 @@ def get_dataset_items(
                             "will be removed.",
                             UserWarning,
                         )
-            annotation_scene = AnnotationSceneEntity(
-                annotations=annotations, kind=AnnotationSceneKind.ANNOTATION
-            )
+            annotation_scene = AnnotationSceneEntity(annotations=annotations, kind=AnnotationSceneKind.ANNOTATION)
 
             # Add to dataset items
-            dataset_items.append(
-                DatasetItemEntity(
-                    media=image, annotation_scene=annotation_scene, subset=subset
-                )
-            )
+            dataset_items.append(DatasetItemEntity(media=image, annotation_scene=annotation_scene, subset=subset))
 
         return dataset_items
 
@@ -268,9 +246,7 @@ class AnomalyDetectionDataset(BaseAnomalyDataset):
 
     """
 
-    def get_dataset_items(
-        self, ann_file_path: Path, data_root_dir: Path, subset: Subset
-    ) -> List[DatasetItemEntity]:
+    def get_dataset_items(self, ann_file_path: Path, data_root_dir: Path, subset: Subset) -> List[DatasetItemEntity]:
         """Loads dataset based on the image path in annotation file.
 
         Args:
@@ -291,9 +267,7 @@ def get_dataset_items(
             # convert path to str as PosixPath is not supported by Image
             image = Image(file_path=str(data_root_dir / sample.image_path))
             # Create annotation
-            label: LabelEntity = (
-                self.normal_label if sample.label == "good" else self.abnormal_label
-            )
+            label: LabelEntity = self.normal_label if sample.label == "good" else self.abnormal_label
             annotations = [
                 Annotation(
                     Rectangle.generate_full_box(),
@@ -320,15 +294,9 @@ def get_dataset_items(
                             "will be removed.",
                             UserWarning,
                         )
-            annotation_scene = AnnotationSceneEntity(
-                annotations=annotations, kind=AnnotationSceneKind.ANNOTATION
-            )
+            annotation_scene = AnnotationSceneEntity(annotations=annotations, kind=AnnotationSceneKind.ANNOTATION)
 
             # Add to dataset items
-            dataset_items.append(
-                DatasetItemEntity(
-                    media=image, annotation_scene=annotation_scene, subset=subset
-                )
-            )
+            dataset_items.append(DatasetItemEntity(media=image, annotation_scene=annotation_scene, subset=subset))
 
         return dataset_items

ote_cli/ote_cli/datasets/anomaly/__init__.py → external/anomaly/tools/__init__.py

@@ -1,6 +1,8 @@
-"""DataLoader for anomaly tasks."""
+"""
+Collection of tools to run anomaly training extension.
+"""
 
-# Copyright (C) 2021 Intel Corporation
+# Copyright (C) 2022 Intel Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

external/anomaly/ote_anomalib/tools/sample.py → external/anomaly/tools/sample.py

@@ -22,16 +22,20 @@
 import os
 import shutil
 from argparse import Namespace
-from typing import Any
+from typing import Any, Dict, Type, Union
 
 from ote_anomalib import AnomalyNNCFTask, OpenVINOAnomalyTask
-from ote_anomalib.data.mvtec import OteMvtecDataset
+from ote_anomalib.data.dataset import (
+    AnomalyClassificationDataset,
+    AnomalyDetectionDataset,
+    AnomalySegmentationDataset,
+)
 from ote_anomalib.logging import get_logger
 from ote_sdk.configuration.helper import create as create_hyper_parameters
 from ote_sdk.entities.inference_parameters import InferenceParameters
 from ote_sdk.entities.label_schema import LabelSchemaEntity
 from ote_sdk.entities.model import ModelEntity
-from ote_sdk.entities.model_template import parse_model_template
+from ote_sdk.entities.model_template import TaskType, parse_model_template
 from ote_sdk.entities.optimization_parameters import OptimizationParameters
 from ote_sdk.entities.resultset import ResultSetEntity
 from ote_sdk.entities.subset import Subset
@@ -50,7 +54,14 @@
 class OteAnomalyTask:
     """OTE Anomaly Classification Task."""
 
-    def __init__(self, dataset_path: str, seed: int, model_template_path: str) -> None:
+    def __init__(
+        self,
+        dataset_path: str,
+        train_subset: Dict[str, str],
+        val_subset: Dict[str, str],
+        test_subset: Dict[str, str],
+        model_template_path: str,
+    ) -> None:
         """Initialize OteAnomalyTask.
 
         Args:
@@ -84,7 +95,10 @@ def __init__(self, dataset_path: str, seed: int, model_template_path: str) -> No
 
         logger.info("Loading MVTec dataset.")
         self.task_type = self.model_template.task_type
-        self.dataset = OteMvtecDataset(path=dataset_path, seed=seed, task_type=self.task_type).generate()
+
+        dataclass = self.get_dataclass()
+
+        self.dataset = dataclass(train_subset, val_subset, test_subset)
 
         logger.info("Creating the task-environment.")
         self.task_environment = self.create_task_environment()
@@ -97,6 +111,27 @@ def __init__(self, dataset_path: str, seed: int, model_template_path: str) -> No
         self.nncf_task: AnomalyNNCFTask
         self.results = {"category": dataset_path}
 
+    def get_dataclass(
+        self,
+    ) -> Union[Type[AnomalyDetectionDataset], Type[AnomalySegmentationDataset], Type[AnomalyClassificationDataset]]:
+        """Gets the dataloader based on the task type.
+
+        Raises:
+            ValueError: Validates task type.
+
+        Returns:
+           Dataloader
+        """
+        if self.task_type == TaskType.ANOMALY_DETECTION:
+            dataclass = AnomalyDetectionDataset
+        elif self.task_type == TaskType.ANOMALY_SEGMENTATION:
+            dataclass = AnomalySegmentationDataset
+        elif self.task_type == TaskType.ANOMALY_CLASSIFICATION:
+            dataclass = AnomalyClassificationDataset
+        else:
+            raise ValueError(f"{self.task_type} not a supported task")
+        return dataclass
+
     def create_task_environment(self) -> TaskEnvironment:
         """Create task environment."""
         hyper_parameters = create_hyper_parameters(self.model_template.hyper_parameters.data)
@@ -306,6 +341,9 @@ def parse_args() -> Namespace:
     )
     parser.add_argument("--dataset_path", default="./datasets/MVTec")
     parser.add_argument("--category", default="bottle")
+    parser.add_argument("--train-ann-files", required=True)
+    parser.add_argument("--val-ann-files", required=True)
+    parser.add_argument("--test-ann-files", required=True)
     parser.add_argument("--optimization", choices=("none", "pot", "nncf"), default="none")
     parser.add_argument("--seed", default=0)
     return parser.parse_args()
@@ -316,7 +354,17 @@ def main() -> None:
     args = parse_args()
     path = os.path.join(args.dataset_path, args.category)
 
-    task = OteAnomalyTask(dataset_path=path, seed=args.seed, model_template_path=args.model_template_path)
+    train_subset = {"ann_file": args.train_ann_files, "data_root": path}
+    val_subset = {"ann_file": args.val_ann_files, "data_root": path}
+    test_subset = {"ann_file": args.test_ann_files, "data_root": path}
+
+    task = OteAnomalyTask(
+        dataset_path=path,
+        train_subset=train_subset,
+        val_subset=val_subset,
+        test_subset=test_subset,
+        model_template_path=args.model_template_path,
+    )
 
     task.train()
     task.export()

external/deep-object-reid/configs/ote_custom_classification/efficientnet_b0/template.yaml

@@ -29,10 +29,12 @@ hyper_parameters:
     learning_parameters:
       batch_size:
         default_value: 32
+        auto_hpo_state: POSSIBLE
       max_num_epochs:
         default_value: 200
       learning_rate:
         default_value: 0.007
+        auto_hpo_state: POSSIBLE
       enable_early_stopping:
         default_value: true
     nncf_optimization:

external/deep-object-reid/configs/ote_custom_classification/efficientnet_v2_s/template.yaml

@@ -29,10 +29,12 @@ hyper_parameters:
     learning_parameters:
       batch_size:
         default_value: 32
+        auto_hpo_state: POSSIBLE
       max_num_epochs:
         default_value: 200
       learning_rate:
         default_value: 0.007
+        auto_hpo_state: POSSIBLE
       enable_early_stopping:
         default_value: true
     nncf_optimization: