Merge branch 'develop' into gz/fix_xai_tv_models

.github/CODEOWNERS

@@ -1,4 +1,4 @@
 # See help here: https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners
 
 # These owners will be the default owners for everything in the repo.
-* @samet-akcay @harimkang @vinnamkim @jaegukhyun @sungmanc @eugene123tw @kprokofi @chuneuny-emily @sovrasov
+* @samet-akcay @harimkang @vinnamkim @jaegukhyun @eugene123tw @kprokofi @chuneuny-emily @sovrasov @sungchul2 @GalyaZalesskaya @negvet @goodsong81 @yunchu @wonjuleee @eunwoosh

.github/workflows/perf_benchmark.yaml

@@ -167,7 +167,7 @@ jobs:
       - name: Install dependencies
         run: python -m pip install --upgrade pip pandas matplotlib nbconvert ipython ipykernel openpyxl
       - name: Download benchmark results
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@87c55149d96e628cc2ef7e6fc2aab372015aec85 # v4.1.3
         with:
           path: tests/perf/history/latest
       - name: Summarize benchamrk results

docs/source/guide/explanation/algorithms/anomaly/index.rst

@@ -153,29 +153,3 @@ Since STFPM trains the student network, we use the following parameters for its
    - ``Early Stopping``: Early stopping is used to stop the training process when the validation loss stops improving. The default value of the early stopping patience is ``10``.
 
 For more information on STFPM's training. We invite you to read Anomalib's `STFPM documentation <https://anomalib.readthedocs.io/en/v1.0.0/markdown/guides/reference/models/image/stfpm.html>`_.
-
-Reconstruction-based Models
----------------------------
-These models initially extract features from a CNN or transformer and subsequently reconstruct the input image. The anomaly score is then calculated as the distance between the input image and the reconstructed image. OpenVINO Training Extensions currently supports `DRÆM – A discriminatively trained reconstruction embedding for surface anomaly detection <https://arxiv.org/pdf/2108.07610v2.pdf>`_.
-
-DRÆM
-^^^^
-
-.. figure:: ../../../../../utils/images/draem.png
-   :width: 600
-   :align: center
-   :alt: Anomaly Task Types
-
-A reconstruction-based algorithm, DRAEM consists of a reconstructive subnetwork and a discriminative subnetwork. DRAEM is trained on simulated anomaly images, which are produced by combining normal input images from the training set with a random Perlin noise mask extracted from an unrelated source of image data. The reconstructive subnetwork is an autoencoder trained to reconstruct the original input images from the augmented images. Combining L2 loss and structural similarity loss, the reconstructive submodel is trained. The input of the discriminative subnetwork is the channel-by-channel concatenation of the (augmented) input image and the output of the reconstructive subnetwork. The output of the discriminative subnetwork is an anomaly map containing the predicted anomaly scores for each pixel.
-
-Training Parameters
-~~~~~~~~~~~~~~~~~~~~
-
-- ``Enable SSPCAB``: The default value is ``False``. If set to ``True``, the model will use the SSPCAB block.
-- ``SSPCAB Lambda``: The default value is ``0.1``. This parameter is used to control the trade-off between the self-supervised reconstruction loss and the training loss.
-- ``Anomaly Source Path``: The default value is ``None``. This parameter is used to specify the path to the anomaly source images.
-- ``Beta``: Parameter that determines the opacity of the noise mask. The default is (0.1, 1.0).
-- ``Optimizer``: Both the reconstructive subnetwork and the discriminative subnetwork are trained using the Adam optimizer.
-- ``Loss``: The reconstructive subnetwork is trained using reconstruction loss which consists of a combination of L2 loss and Structural Similarity (SSIM) loss between the reconstructions and the original images. The discriminative subnetwork is trained using focal loss, computed between the pixel-level predictions and the ground truth masks of the augmented images.
-- ``Additional Training Techniques``:
-   - ``Early Stopping``: Early stopping is used to prevent overfitting. The early stopping patience can be configured by the user. By default, early stopping is enabled with a patience of 20 epochs.

docs/source/guide/get_started/installation.rst

@@ -69,7 +69,7 @@ Follow the `official documentation <https://intel.github.io/intel-extension-for-
 
 .. code-block:: shell
 
-    python -m pip install torch==2.1.0a0 torchvision==0.16.0a0 torchaudio==2.1.0a0 intel-extension-for-pytorch==2.1.10+xpu --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/
+    python -m pip install torch==2.1.0.post0 torchvision==0.16.0.post0 torchaudio==2.1.0.post0 intel-extension-for-pytorch==2.1.20+xpu oneccl_bind_pt==2.1.200+xpu --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/
 
 3. Install MMCV.
 It is required to install mmcv from source to properly build it with IPEX.

pyproject.toml

@@ -234,6 +234,12 @@ ignore = [
 
     "E731",     # Do not assign a `lambda` expression, use a `def`
     "TD003",    # Missing issue link on the line following this TODO
+
+    # TODO (someone): Update legacy np.random.{method_name} call with np.random.Generator
+    # Legacy (MT19964) and new (PCG64) random generators use different pseudo-random number generators.
+    # (https://numpy.org/doc/stable/reference/random/legacy.html#legacy)
+    # For now, remaining legacy random methods is required until numpy version is explicitly updated in OTX.
+    "NPY002",   # Replace legacy np.random.{method_name} call with np.random.Generator
 ]
 
 # Allow autofix for all enabled rules (when `--fix`) is provided.

src/otx/algo/anomaly/__init__.py

@@ -3,8 +3,7 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-from .draem import Draem
 from .padim import Padim
 from .stfpm import Stfpm
 
-__all__ = ["Draem", "Padim", "Stfpm"]
+__all__ = ["Padim", "Stfpm"]

src/otx/algo/classification/mobilenet_v3.py

@@ -9,7 +9,7 @@
 from typing import TYPE_CHECKING, Any, Callable, Literal
 
 import torch
-from torch import nn
+from torch import Tensor, nn
 
 from otx.algo.classification.backbones import OTXMobileNetV3
 from otx.algo.classification.classifier.base_classifier import ImageClassifier
@@ -158,14 +158,12 @@ def forward_explain(self, inputs: MulticlassClsBatchDataEntity) -> MulticlassCls
             feature_vector=outputs["feature_vector"],
         )
 
-    def _reset_model_forward(self) -> None:
-        # TODO(vinnamkim): This will be revisited by the export refactoring
-        self.__orig_model_forward = self.model.forward
-        self.model.forward = self.model._forward_explain  # type: ignore[assignment] # noqa: SLF001
+    def forward_for_tracing(self, image: Tensor) -> Tensor | dict[str, Tensor]:
+        """Model forward function used for the model tracing during model exportation."""
+        if self.explain_mode:
+            return self.model(images=image, mode="explain")
 
-    def _restore_model_forward(self) -> None:
-        # TODO(vinnamkim): This will be revisited by the export refactoring
-        self.model.forward = self.__orig_model_forward  # type: ignore[method-assign]
+        return self.model(images=image, mode="tensor")
 
 
 class MobileNetV3ForMultilabelCls(OTXMultilabelClsModel):
@@ -277,14 +275,12 @@ def forward_explain(self, inputs: MultilabelClsBatchDataEntity) -> MultilabelCls
             feature_vector=outputs["feature_vector"],
         )
 
-    def _reset_model_forward(self) -> None:
-        # TODO(vinnamkim): This will be revisited by the export refactoring
-        self.__orig_model_forward = self.model.forward
-        self.model.forward = self.model._forward_explain  # type: ignore[assignment] # noqa: SLF001
+    def forward_for_tracing(self, image: Tensor) -> Tensor | dict[str, Tensor]:
+        """Model forward function used for the model tracing during model exportation."""
+        if self.explain_mode:
+            return self.model(images=image, mode="explain")
 
-    def _restore_model_forward(self) -> None:
-        # TODO(vinnamkim): This will be revisited by the export refactoring
-        self.model.forward = self.__orig_model_forward  # type: ignore[method-assign]
+        return self.model(images=image, mode="tensor")
 
 
 class MobileNetV3ForHLabelCls(OTXHlabelClsModel):
@@ -423,11 +419,9 @@ def forward_explain(self, inputs: HlabelClsBatchDataEntity) -> HlabelClsBatchPre
             feature_vector=outputs["feature_vector"],
         )
 
-    def _reset_model_forward(self) -> None:
-        # TODO(vinnamkim): This will be revisited by the export refactoring
-        self.__orig_model_forward = self.model.forward
-        self.model.forward = self.model._forward_explain  # type: ignore[assignment] # noqa: SLF001
+    def forward_for_tracing(self, image: Tensor) -> Tensor | dict[str, Tensor]:
+        """Model forward function used for the model tracing during model exportation."""
+        if self.explain_mode:
+            return self.model(images=image, mode="explain")
 
-    def _restore_model_forward(self) -> None:
-        # TODO(vinnamkim): This will be revisited by the export refactoring
-        self.model.forward = self.__orig_model_forward  # type: ignore[method-assign]
+        return self.model(images=image, mode="tensor")

src/otx/algo/classification/otx_dino_v2.py

@@ -8,7 +8,7 @@
 from typing import TYPE_CHECKING, Any
 
 import torch
-from torch import nn
+from torch import Tensor, nn
 
 from otx.algo.utils.mmconfig import read_mmconfig
 from otx.core.data.entity.base import OTXBatchLossEntity
@@ -162,3 +162,7 @@ def _exporter(self) -> OTXModelExporter:
     def _optimization_config(self) -> dict[str, Any]:
         """PTQ config for DinoV2Cls."""
         return {"model_type": "transformer"}
+
+    def forward_for_tracing(self, image: Tensor) -> Tensor | dict[str, Tensor]:
+        """Model forward function used for the model tracing during model exportation."""
+        return self.model(image)

src/otx/algo/classification/torchvision_model.py

@@ -8,7 +8,7 @@
 from typing import TYPE_CHECKING, Any, Callable, Literal
 
 import torch
-from torch import nn
+from torch import Tensor, nn
 from torchvision.models import get_model, get_model_weights
 
 from otx.algo.explain.explain_algo import ReciproCAM, feature_vector_fn
@@ -320,11 +320,9 @@ def forward_explain(self, inputs: MulticlassClsBatchDataEntity) -> MulticlassCls
             feature_vector=outputs["feature_vector"],
         )
 
-    def _reset_model_forward(self) -> None:
-        # TODO(vinnamkim): This will be revisited by the export refactoring
-        self.__orig_model_forward = self.model.forward
-        self.model.forward = self.model._forward_explain  # type: ignore[assignment] # noqa: SLF001
+    def forward_for_tracing(self, image: Tensor) -> Tensor | dict[str, Tensor]:
+        """Model forward function used for the model tracing during model exportation."""
+        if self.explain_mode:
+            return self.model(images=image, mode="explain")
 
-    def _restore_model_forward(self) -> None:
-        # TODO(vinnamkim): This will be revisited by the export refactoring
-        self.model.forward = self.__orig_model_forward  # type: ignore[method-assign]
+        return self.model(images=image, mode="tensor")

src/otx/algo/detection/atss.py

@@ -57,20 +57,21 @@ def _exporter(self) -> OTXModelExporter:
 
         mean, std = get_mean_std_from_data_processing(self.config)
 
-        return MMdeployExporter(
-            model_builder=self._create_model,
-            model_cfg=deepcopy(self.config),
-            deploy_cfg="otx.algo.detection.mmdeploy.atss",
-            test_pipeline=self._make_fake_test_pipeline(),
-            task_level_export_parameters=self._export_parameters,
-            input_size=self.image_size,
-            mean=mean,
-            std=std,
-            resize_mode="standard",
-            pad_value=0,
-            swap_rgb=False,
-            output_names=["feature_vector", "saliency_map"] if self.explain_mode else None,
-        )
+        with self.export_model_forward_context():
+            return MMdeployExporter(
+                model_builder=self._create_model,
+                model_cfg=deepcopy(self.config),
+                deploy_cfg="otx.algo.detection.mmdeploy.atss",
+                test_pipeline=self._make_fake_test_pipeline(),
+                task_level_export_parameters=self._export_parameters,
+                input_size=self.image_size,
+                mean=mean,
+                std=std,
+                resize_mode="standard",
+                pad_value=0,
+                swap_rgb=False,
+                output_names=["feature_vector", "saliency_map"] if self.explain_mode else None,
+            )
 
     def load_from_otx_v1_ckpt(self, state_dict: dict, add_prefix: str = "model.model.") -> dict:
         """Load the previous OTX ckpt according to OTX2.0."""

src/otx/algo/detection/rtmdet.py

@@ -57,20 +57,21 @@ def _exporter(self) -> OTXModelExporter:
 
         mean, std = get_mean_std_from_data_processing(self.config)
 
-        return MMdeployExporter(
-            model_builder=self._create_model,
-            model_cfg=deepcopy(self.config),
-            deploy_cfg="otx.algo.detection.mmdeploy.rtmdet",
-            test_pipeline=self._make_fake_test_pipeline(),
-            task_level_export_parameters=self._export_parameters,
-            input_size=self.image_size,
-            mean=mean,
-            std=std,
-            resize_mode="fit_to_window_letterbox",
-            pad_value=114,
-            swap_rgb=False,
-            output_names=["feature_vector", "saliency_map"] if self.explain_mode else None,
-        )
+        with self.export_model_forward_context():
+            return MMdeployExporter(
+                model_builder=self._create_model,
+                model_cfg=deepcopy(self.config),
+                deploy_cfg="otx.algo.detection.mmdeploy.rtmdet",
+                test_pipeline=self._make_fake_test_pipeline(),
+                task_level_export_parameters=self._export_parameters,
+                input_size=self.image_size,
+                mean=mean,
+                std=std,
+                resize_mode="fit_to_window_letterbox",
+                pad_value=114,
+                swap_rgb=False,
+                output_names=["feature_vector", "saliency_map"] if self.explain_mode else None,
+            )
 
     def load_from_otx_v1_ckpt(self, state_dict: dict, add_prefix: str = "model.model.") -> dict:
         """Load the previous OTX ckpt according to OTX2.0."""