From 8fafd40ad0879f7d4c0d480ee67eaae21c5bc23c Mon Sep 17 00:00:00 2001 From: Ashwin Vaidya Date: Tue, 24 Oct 2023 02:58:20 +0200 Subject: [PATCH] Add Anomaly modelAPI changes to releases/1.4.0 (#2563) * bug fix for legacy openvino models * Apply otx anomaly 1.5 changes * Fix tests * Fix compression config * fix modelAPI imports * update integration tests * Edit config types * Update keys in deployed model --------- Co-authored-by: Ashwin Vaidya Co-authored-by: Kim, Sungchul --- requirements/openvino.txt | 2 +- .../model_wrappers/openvino_models.py | 7 +- .../anomalib/exportable_code/__init__.py | 12 -- .../exportable_code/anomaly_classification.py | 47 ----- .../exportable_code/anomaly_detection.py | 43 ----- .../exportable_code/anomaly_segmentation.py | 43 ----- .../adapters/anomalib/exportable_code/base.py | 47 ----- src/otx/algorithms/anomaly/tasks/inference.py | 51 ++++-- src/otx/algorithms/anomaly/tasks/openvino.py | 166 ++++++++++++------ src/otx/algorithms/anomaly/tasks/train.py | 2 +- src/otx/algorithms/common/utils/ir.py | 4 +- src/otx/algorithms/common/utils/utils.py | 16 +- .../configs/base/configuration.py | 8 +- .../demo/demo_package/model_container.py | 18 +- .../exportable_code/demo/requirements.txt | 2 +- .../prediction_to_annotation_converter.py | 44 +++-- .../compressed_model.yml | 2 + .../compressed_model.yml | 2 + .../compressed_model.yml | 2 + .../anomaly/test_anomaly_classification.py | 4 +- .../e2e/cli/anomaly/test_anomaly_detection.py | 4 +- .../cli/anomaly/test_anomaly_segmentation.py | 4 +- .../anomaly/test_anomaly_classification.py | 6 +- .../cli/anomaly/test_anomaly_detection.py | 6 +- .../cli/anomaly/test_anomaly_segmentation.py | 6 +- .../algorithms/anomaly/tasks/test_openvino.py | 2 +- ...test_prediction_to_annotation_converter.py | 66 ------- 27 files changed, 238 insertions(+), 378 deletions(-) delete mode 100644 src/otx/algorithms/anomaly/adapters/anomalib/exportable_code/__init__.py delete mode 100644 src/otx/algorithms/anomaly/adapters/anomalib/exportable_code/anomaly_classification.py delete mode 100644 src/otx/algorithms/anomaly/adapters/anomalib/exportable_code/anomaly_detection.py delete mode 100644 src/otx/algorithms/anomaly/adapters/anomalib/exportable_code/anomaly_segmentation.py delete mode 100644 src/otx/algorithms/anomaly/adapters/anomalib/exportable_code/base.py diff --git a/requirements/openvino.txt b/requirements/openvino.txt index 6424a9b6778..4a1494bb460 100644 --- a/requirements/openvino.txt +++ b/requirements/openvino.txt @@ -2,7 +2,7 @@ # OpenVINO Requirements. # nncf==2.6.0 onnx==1.13.0 -openvino-model-api==0.1.3 +openvino-model-api==0.1.6 openvino==2023.0 openvino-dev==2023.0 openvino-telemetry>=2022.1.0 diff --git a/src/otx/algorithms/action/adapters/openvino/model_wrappers/openvino_models.py b/src/otx/algorithms/action/adapters/openvino/model_wrappers/openvino_models.py index dcbfe71a1dc..63a1492c20e 100644 --- a/src/otx/algorithms/action/adapters/openvino/model_wrappers/openvino_models.py +++ b/src/otx/algorithms/action/adapters/openvino/model_wrappers/openvino_models.py @@ -20,12 +20,9 @@ import numpy as np from openvino.model_api.adapters import OpenvinoAdapter +from openvino.model_api.adapters.utils import RESIZE_TYPES, InputTransform from openvino.model_api.models.model import Model -from openvino.model_api.models.utils import ( - RESIZE_TYPES, - Detection, - InputTransform, -) +from openvino.model_api.models.utils import Detection from otx.api.entities.datasets import DatasetItemEntity diff --git a/src/otx/algorithms/anomaly/adapters/anomalib/exportable_code/__init__.py b/src/otx/algorithms/anomaly/adapters/anomalib/exportable_code/__init__.py deleted file mode 100644 index 5bcf71d66ca..00000000000 --- a/src/otx/algorithms/anomaly/adapters/anomalib/exportable_code/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Exportable code for Anomaly tasks.""" - -# Copyright (C) 2021-2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -from .anomaly_classification import AnomalyClassification -from .anomaly_detection import AnomalyDetection -from .anomaly_segmentation import AnomalySegmentation -from .base import AnomalyBase - -__all__ = ["AnomalyBase", "AnomalyClassification", "AnomalyDetection", "AnomalySegmentation"] diff --git a/src/otx/algorithms/anomaly/adapters/anomalib/exportable_code/anomaly_classification.py b/src/otx/algorithms/anomaly/adapters/anomalib/exportable_code/anomaly_classification.py deleted file mode 100644 index bd61d0f3c05..00000000000 --- a/src/otx/algorithms/anomaly/adapters/anomalib/exportable_code/anomaly_classification.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Wrapper for Open Model Zoo for Anomaly Classification tasks.""" - -# Copyright (C) 2021-2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -from typing import Any, Dict - -import cv2 -import numpy as np - -from .base import AnomalyBase - - -class AnomalyClassification(AnomalyBase): - """Wrapper for anomaly classification task.""" - - __model__ = "anomaly_classification" - - def postprocess(self, outputs: Dict[str, np.ndarray], meta: Dict[str, Any]) -> float: - """Resize the outputs of the model to original image size. - - Args: - outputs (Dict[str, np.ndarray]): Raw outputs of the model after ``infer_sync`` is called. - meta (Dict[str, Any]): Metadata which contains values such as threshold, original image size. - - Returns: - float: Normalized anomaly score - """ - anomaly_map: np.ndarray = outputs[self.output_blob_name].squeeze() - pred_score = anomaly_map.reshape(-1).max() - - meta["image_threshold"] = self.metadata["image_threshold"] # pylint: disable=no-member - meta["min"] = self.metadata["min"] # pylint: disable=no-member - meta["max"] = self.metadata["max"] # pylint: disable=no-member - meta["threshold"] = self.threshold # pylint: disable=no-member - - anomaly_map = self._normalize(anomaly_map, meta["image_threshold"], meta["min"], meta["max"]) - pred_score = self._normalize(pred_score, meta["image_threshold"], meta["min"], meta["max"]) - - input_image_height = meta["original_shape"][0] - input_image_width = meta["original_shape"][1] - result = cv2.resize(anomaly_map, (input_image_width, input_image_height)) - - meta["anomaly_map"] = result - - return np.array(pred_score) diff --git a/src/otx/algorithms/anomaly/adapters/anomalib/exportable_code/anomaly_detection.py b/src/otx/algorithms/anomaly/adapters/anomalib/exportable_code/anomaly_detection.py deleted file mode 100644 index 47e8b49697e..00000000000 --- a/src/otx/algorithms/anomaly/adapters/anomalib/exportable_code/anomaly_detection.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Wrapper for Open Model Zoo for Anomaly Detection tasks.""" - -# Copyright (C) 2021-2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -from typing import Any, Dict - -import cv2 -import numpy as np - -from .base import AnomalyBase - - -class AnomalyDetection(AnomalyBase): - """Wrapper for anomaly detection task.""" - - __model__ = "anomaly_detection" - - def postprocess(self, outputs: Dict[str, np.ndarray], meta: Dict[str, Any]) -> np.ndarray: - """Resize the outputs of the model to original image size. - - Args: - outputs (Dict[str, np.ndarray]): Raw outputs of the model after ``infer_sync`` is called. - meta (Dict[str, Any]): Metadata which contains values such as threshold, original image size. - - Returns: - np.ndarray: Detection Mask - """ - anomaly_map: np.ndarray = outputs[self.output_blob_name].squeeze() - - meta["pixel_threshold"] = self.metadata["pixel_threshold"] # pylint: disable=no-member - meta["min"] = self.metadata["min"] # pylint: disable=no-member - meta["max"] = self.metadata["max"] # pylint: disable=no-member - meta["threshold"] = self.threshold # pylint: disable=no-member - - anomaly_map = self._normalize(anomaly_map, meta["pixel_threshold"], meta["min"], meta["max"]) - - input_image_height = meta["original_shape"][0] - input_image_width = meta["original_shape"][1] - result = cv2.resize(anomaly_map, (input_image_width, input_image_height)) - - return result diff --git a/src/otx/algorithms/anomaly/adapters/anomalib/exportable_code/anomaly_segmentation.py b/src/otx/algorithms/anomaly/adapters/anomalib/exportable_code/anomaly_segmentation.py deleted file mode 100644 index 7335e914024..00000000000 --- a/src/otx/algorithms/anomaly/adapters/anomalib/exportable_code/anomaly_segmentation.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Wrapper for Open Model Zoo for Anomaly Segmentation tasks.""" - -# Copyright (C) 2021-2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -from typing import Any, Dict - -import cv2 -import numpy as np - -from .base import AnomalyBase - - -class AnomalySegmentation(AnomalyBase): - """Wrapper for anomaly segmentation task.""" - - __model__ = "anomaly_segmentation" - - def postprocess(self, outputs: Dict[str, np.ndarray], meta: Dict[str, Any]) -> np.ndarray: - """Resize the outputs of the model to original image size. - - Args: - outputs (Dict[str, np.ndarray]): Raw outputs of the model after ``infer_sync`` is called. - meta (Dict[str, Any]): Metadata which contains values such as threshold, original image size. - - Returns: - np.ndarray: Segmentation Mask - """ - anomaly_map: np.ndarray = outputs[self.output_blob_name].squeeze() - - meta["pixel_threshold"] = self.metadata["pixel_threshold"] # pylint: disable=no-member - meta["min"] = self.metadata["min"] # pylint: disable=no-member - meta["max"] = self.metadata["max"] # pylint: disable=no-member - meta["threshold"] = self.threshold # pylint: disable=no-member - - anomaly_map = self._normalize(anomaly_map, meta["pixel_threshold"], meta["min"], meta["max"]) - - input_image_height = meta["original_shape"][0] - input_image_width = meta["original_shape"][1] - result = cv2.resize(anomaly_map, (input_image_width, input_image_height)) - - return result diff --git a/src/otx/algorithms/anomaly/adapters/anomalib/exportable_code/base.py b/src/otx/algorithms/anomaly/adapters/anomalib/exportable_code/base.py deleted file mode 100644 index cf25d59fff2..00000000000 --- a/src/otx/algorithms/anomaly/adapters/anomalib/exportable_code/base.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Wrapper for Open Model Zoo for Anomaly tasks.""" - -# Copyright (C) 2021-2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -from typing import Union - -import numpy as np -from openvino.model_api.models import SegmentationModel -from openvino.model_api.models.types import DictValue, NumericalValue - - -class AnomalyBase(SegmentationModel): - """Wrapper for anomaly tasks.""" - - __model__ = "anomaly_base" - - @classmethod - def parameters(cls): - """Dictionary containing model parameters.""" - parameters = super().parameters() - parameters["resize_type"].update_default_value("standard") - parameters.update( - { - "metadata": DictValue(description="Metadata for inference"), - "threshold": NumericalValue(description="Threshold used to classify anomaly"), - } - ) - - return parameters - - @staticmethod - def _normalize( - targets: Union[np.ndarray, np.float32], - threshold: Union[np.ndarray, float], - min_val: Union[np.ndarray, float], - max_val: Union[np.ndarray, float], - ) -> np.ndarray: - """Apply min-max normalization and shift the values such that the threshold value is centered at 0.5.""" - normalized = ((targets - threshold) / (max_val - min_val)) + 0.5 - if isinstance(targets, (np.ndarray, np.float32)): - normalized = np.minimum(normalized, 1) - normalized = np.maximum(normalized, 0) - else: - raise ValueError(f"Targets must be either Tensor or Numpy array. Received {type(targets)}") - return normalized diff --git a/src/otx/algorithms/anomaly/tasks/inference.py b/src/otx/algorithms/anomaly/tasks/inference.py index 81240f31217..50c5a4b81f7 100644 --- a/src/otx/algorithms/anomaly/tasks/inference.py +++ b/src/otx/algorithms/anomaly/tasks/inference.py @@ -22,7 +22,6 @@ import subprocess # nosec B404 import tempfile from glob import glob -from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Union from warnings import warn @@ -36,7 +35,6 @@ PostProcessingConfigurationCallback, ) from omegaconf import DictConfig, ListConfig -from openvino.runtime import Core, serialize from pytorch_lightning import Trainer from otx.algorithms.anomaly.adapters.anomalib.callbacks import ( @@ -47,6 +45,8 @@ from otx.algorithms.anomaly.adapters.anomalib.data import OTXAnomalyDataModule from otx.algorithms.anomaly.adapters.anomalib.logger import get_logger from otx.algorithms.anomaly.configs.base.configuration import BaseAnomalyConfig +from otx.algorithms.common.utils import embed_ir_model_data +from otx.algorithms.common.utils.utils import embed_onnx_model_data from otx.api.entities.datasets import DatasetEntity from otx.api.entities.inference_parameters import InferenceParameters from otx.api.entities.metrics import NullPerformance, Performance, ScoreMetric @@ -296,6 +296,8 @@ def export( self._export_to_onnx(onnx_path) if export_type == ExportType.ONNX: + self._add_metadata_to_ir(onnx_path, export_type) + with open(onnx_path, "rb") as file: output_model.set_data("model.onnx", file.read()) else: @@ -306,7 +308,7 @@ def export( bin_file = glob(os.path.join(self.config.project.path, "*.bin"))[0] xml_file = glob(os.path.join(self.config.project.path, "*.xml"))[0] - self._add_metadata_to_ir(xml_file) + self._add_metadata_to_ir(xml_file, export_type) with open(bin_file, "rb") as file: output_model.set_data("openvino.bin", file.read()) @@ -319,40 +321,51 @@ def export( output_model.set_data("label_schema.json", label_schema_to_bytes(self.task_environment.label_schema)) self._set_metadata(output_model) - def _add_metadata_to_ir(self, xml_file: str) -> None: - """Adds the metadata to the model IR. + def _add_metadata_to_ir(self, model_file: str, export_type: ExportType) -> None: + """Adds the metadata to the model IR or ONNX. Adds the metadata to the model IR. So that it can be used with the new modelAPI. This is because the metadata.json is not used by the new modelAPI. # TODO CVS-114640 # TODO: Step 1. Remove metadata.json when modelAPI becomes the default inference method. - # TODO: Step 2. Remove this function when Anomalib is upgraded as the model graph will contain the required ops + # TODO: Step 2. Update this function when Anomalib is upgraded as the model graph will contain the required ops # TODO: Step 3. Update modelAPI to remove pre/post-processing steps when Anomalib version is upgraded. """ metadata = self._get_metadata_dict() - core = Core() - model = core.read_model(xml_file) + extra_model_data: Dict[Tuple[str, str], Any] = {} for key, value in metadata.items(): - if key == "transform": + if key in ("transform", "min", "max"): continue - model.set_rt_info(value, ["model_info", key]) + extra_model_data[("model_info", key)] = value # Add transforms if "transform" in metadata: for transform_dict in metadata["transform"]["transform"]["transforms"]: transform = transform_dict.pop("__class_fullname__") if transform == "Normalize": - model.set_rt_info(self._serialize_list(transform_dict["mean"]), ["model_info", "mean_values"]) - model.set_rt_info(self._serialize_list(transform_dict["std"]), ["model_info", "scale_values"]) + extra_model_data[("model_info", "mean_values")] = self._serialize_list( + [x * 255.0 for x in transform_dict["mean"]] + ) + extra_model_data[("model_info", "scale_values")] = self._serialize_list( + [x * 255.0 for x in transform_dict["std"]] + ) elif transform == "Resize": - model.set_rt_info(transform_dict["height"], ["model_info", "orig_height"]) - model.set_rt_info(transform_dict["width"], ["model_info", "orig_width"]) + extra_model_data[("model_info", "orig_height")] = transform_dict["height"] + extra_model_data[("model_info", "orig_width")] = transform_dict["width"] else: warn(f"Transform {transform} is not supported currently") - model.set_rt_info("AnomalyDetection", ["model_info", "model_type"]) - tmp_xml_path = Path(Path(xml_file).parent) / "tmp.xml" - serialize(model, str(tmp_xml_path)) - tmp_xml_path.rename(xml_file) - Path(str(tmp_xml_path.parent / tmp_xml_path.stem) + ".bin").unlink() + # Since we only need the diff of max and min, we fuse the min and max into one op + if "min" in metadata and "max" in metadata: + extra_model_data[("model_info", "normalization_scale")] = metadata["max"] - metadata["min"] + + extra_model_data[("model_info", "reverse_input_channels")] = False + extra_model_data[("model_info", "model_type")] = "AnomalyDetection" + extra_model_data[("model_info", "labels")] = "Normal Anomaly" + if export_type == ExportType.OPENVINO: + embed_ir_model_data(model_file, extra_model_data) + elif export_type == ExportType.ONNX: + embed_onnx_model_data(model_file, extra_model_data) + else: + raise RuntimeError(f"not supported export type {export_type}") def _serialize_list(self, arr: Union[Tuple, List]) -> str: """Converts a list to space separated string.""" diff --git a/src/otx/algorithms/anomaly/tasks/openvino.py b/src/otx/algorithms/anomaly/tasks/openvino.py index 3800b264e0d..f607a4a5d4e 100644 --- a/src/otx/algorithms/anomaly/tasks/openvino.py +++ b/src/otx/algorithms/anomaly/tasks/openvino.py @@ -19,7 +19,7 @@ import os import random import tempfile -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional, Tuple, Union from zipfile import ZipFile import nncf @@ -27,14 +27,14 @@ import openvino.runtime as ov from addict import Dict as ADDict from anomalib.data.utils.transform import get_transforms -from anomalib.deploy import OpenVINOInferencer from nncf.common.quantization.structs import QuantizationPreset from omegaconf import OmegaConf +from openvino.model_api.models import AnomalyDetection, AnomalyResult -import otx.algorithms.anomaly.adapters.anomalib.exportable_code from otx.algorithms.anomaly.adapters.anomalib.config import get_anomalib_config from otx.algorithms.anomaly.adapters.anomalib.logger import get_logger from otx.algorithms.anomaly.configs.base.configuration import BaseAnomalyConfig +from otx.algorithms.common.utils import embed_ir_model_data from otx.algorithms.common.utils.ir import check_if_quantized from otx.algorithms.common.utils.utils import read_py_config from otx.api.configuration.configurable_parameters import ConfigurableParameters @@ -75,22 +75,23 @@ logger = get_logger(__name__) -class OTXOpenVINOAnomalyDataloader: - """Dataloader for loading OTX dataset into OTX OpenVINO Inferencer. +class OTXNNCFAnomalyDataloader: + """Dataloader for loading OTX dataset for NNCF optimization. Args: dataset (DatasetEntity): OTX dataset entity - inferencer (OpenVINOInferencer): OpenVINO Inferencer + model: (AnomalyDetection) The modelAPI model used for fetching the transforms. + shuffle (bool, optional): Shuffle dataset. Defaults to True. """ def __init__( self, dataset: DatasetEntity, - inferencer: OpenVINOInferencer, + model: AnomalyDetection, shuffle: bool = True, ): self.dataset = dataset - self.inferencer = inferencer + self.model = model self.shuffler = None if shuffle: self.shuffler = list(range(len(dataset))) @@ -110,9 +111,12 @@ def __getitem__(self, index: int): image = self.dataset[index].numpy annotation = self.dataset[index].annotation_scene - inputs = self.inferencer.pre_process(image) - return (index, annotation), inputs + resized_image = self.model.resize(image, (self.model.w, self.model.h)) + resized_image = self.model.input_transform(resized_image) + resized_image = self.model._change_layout(resized_image) + + return (index, annotation), resized_image def __len__(self) -> int: """Get size of the dataset. @@ -135,7 +139,7 @@ def __init__(self, task_environment: TaskEnvironment) -> None: self.task_environment = task_environment self.task_type = self.task_environment.model_template.task_type self.config = self.get_config() - self.inferencer = self.load_inferencer() + self.inference_model = self.get_openvino_model() labels = self.task_environment.get_labels() self.normal_label = [label for label in labels if not label.is_anomalous][0] @@ -173,15 +177,13 @@ def infer(self, dataset: DatasetEntity, inference_parameters: InferenceParameter if inference_parameters is not None: update_progress_callback = inference_parameters.update_progress # type: ignore - # This always assumes that threshold is available in the task environment's model - meta_data = self.get_metadata() for idx, dataset_item in enumerate(dataset): - image_result = self.inferencer.predict(dataset_item.numpy, metadata=meta_data) + image_result: AnomalyResult = self.inference_model(dataset_item.numpy) # TODO: inferencer should return predicted label and mask - pred_label = image_result.pred_score >= 0.5 - pred_mask = (image_result.anomaly_map >= 0.5).astype(np.uint8) - probability = image_result.pred_score if pred_label else 1 - image_result.pred_score + pred_label = image_result.pred_label + pred_mask = image_result.pred_mask + probability = image_result.pred_score if pred_label == "Anomaly" else 1 - image_result.pred_score if self.task_type == TaskType.ANOMALY_CLASSIFICATION: label = self.anomalous_label if image_result.pred_score >= 0.5 else self.normal_label elif self.task_type == TaskType.ANOMALY_SEGMENTATION: @@ -320,7 +322,7 @@ def optimize( ) logger.info("Starting PTQ optimization.") - data_loader = OTXOpenVINOAnomalyDataloader(dataset=dataset, inferencer=self.inferencer) + data_loader = OTXNNCFAnomalyDataloader(dataset=dataset, model=self.inference_model) quantization_dataset = nncf.Dataset(data_loader, lambda data: data[1]) with tempfile.TemporaryDirectory() as tempdir: @@ -355,34 +357,105 @@ def optimize( self.__load_weights(path=os.path.join(tempdir, "model.bin"), output_model=output_model, key="openvino.bin") output_model.set_data("label_schema.json", label_schema_to_bytes(self.task_environment.label_schema)) - output_model.set_data("metadata", self.task_environment.model.get_data("metadata")) output_model.model_format = ModelFormat.OPENVINO output_model.optimization_type = ModelOptimizationType.POT output_model.optimization_methods = [OptimizationMethod.QUANTIZATION] output_model.precision = [ModelPrecision.INT8] self.task_environment.model = output_model - self.inferencer = self.load_inferencer() + self.inference_model = self.get_openvino_model() if optimization_parameters is not None: optimization_parameters.update_progress(100, None) logger.info("PTQ optimization completed") - def load_inferencer(self) -> OpenVINOInferencer: + def get_openvino_model(self) -> AnomalyDetection: """Create the OpenVINO inferencer object. Returns: - OpenVINOInferencer object + AnomalyDetection model """ if self.task_environment.model is None: raise Exception("task_environment.model is None. Cannot load weights.") - return OpenVINOInferencer( - path=( - self.task_environment.model.get_data("openvino.xml"), - self.task_environment.model.get_data("openvino.bin"), - ), - metadata=self.get_metadata(), - ) + try: + model = AnomalyDetection.create_model( + model=self.task_environment.model.get_data("openvino.xml"), + weights_path=self.task_environment.model.get_data("openvino.bin"), + ) + except RuntimeError as exception: + logger.exception(exception) + logger.info("Possibly a legacy model is being loaded.") + self._create_from_legacy() + model = AnomalyDetection.create_model( + model=self.task_environment.model.get_data("openvino.xml"), + weights_path=self.task_environment.model.get_data("openvino.bin"), + ) + + return model + + def _create_from_legacy(self) -> None: + """Generates an OpenVINO model in new format from the legacy model. + + TODO: This needs to be removed once all projects in Geti have been migrated to the newer version. + + Args: + model_file (str): The XML model file. + """ + extra_model_data = self._metadata_in_ir_format() + + for key, value in extra_model_data.items(): + if isinstance(value, np.ndarray): + extra_model_data[key] = value.tolist() + + with tempfile.TemporaryDirectory() as temp_dir: + xml_data = self.task_environment.model.get_data("openvino.xml") + bin_data = self.task_environment.model.get_data("openvino.bin") + with open(f"{temp_dir}/openvino.xml", "wb") as file: + file.write(xml_data) + with open(f"{temp_dir}/openvino.bin", "wb") as file: + file.write(bin_data) + embed_ir_model_data(f"{temp_dir}/openvino.xml", extra_model_data) + with open(f"{temp_dir}/openvino.xml", "rb") as file: + self.task_environment.model.set_data("openvino.xml", file.read()) + with open(f"{temp_dir}/openvino.bin", "rb") as file: + self.task_environment.model.set_data("openvino.bin", file.read()) + + def _metadata_in_ir_format(self) -> Dict[Tuple[str, str], Union[str, int, float, List[Union[int, float]]]]: + """Return metadata in format of tuple keys that are used in IR with modelAPI.""" + metadata = self.get_metadata() + extra_model_data: Dict[Tuple[str, str], Any] = {} + for key, value in metadata.items(): + if key in ("transform", "min", "max"): + continue + extra_model_data[("model_info", key)] = value + # Add transforms + if "transform" in metadata: + for transform_dict in metadata["transform"]["transform"]["transforms"]: + transform = transform_dict.pop("__class_fullname__") + if transform == "Normalize": + extra_model_data[("model_info", "mean_values")] = self._serialize_list( + [x * 255.0 for x in transform_dict["mean"]] + ) + extra_model_data[("model_info", "scale_values")] = self._serialize_list( + [x * 255.0 for x in transform_dict["std"]] + ) + elif transform == "Resize": + extra_model_data[("model_info", "orig_height")] = transform_dict["height"] + extra_model_data[("model_info", "orig_width")] = transform_dict["width"] + else: + logger.warn(f"Transform {transform} is not supported currently") + # Since we only need the diff of max and min, we fuse the min and max into one op + if "min" in metadata and "max" in metadata: + extra_model_data[("model_info", "normalization_scale")] = metadata["max"] - metadata["min"] + + extra_model_data[("model_info", "reverse_input_channels")] = False + extra_model_data[("model_info", "model_type")] = "AnomalyDetection" + extra_model_data[("model_info", "labels")] = "Normal Anomaly" + return extra_model_data + + def _serialize_list(self, arr: Union[Tuple, List]) -> str: + """Converts a list to space separated string.""" + return " ".join(map(str, arr)) @staticmethod def __save_weights(path: str, data: bytes) -> None: @@ -412,18 +485,20 @@ def _get_openvino_configuration(self) -> Dict[str, Any]: if self.task_environment.model is None: raise Exception("task_environment.model is None. Cannot get configuration.") - configuration = { - "metadata": self.get_metadata(), + configuration: Dict[str, Any] = { "labels": LabelSchemaMapper.forward(self.task_environment.label_schema), - "threshold": 0.5, } - - if "transforms" not in self.config.keys(): - configuration["mean_values"] = list(np.array([0.485, 0.456, 0.406]) * 255) - configuration["scale_values"] = list(np.array([0.229, 0.224, 0.225]) * 255) - else: - configuration["mean_values"] = self.config.transforms.mean - configuration["scale_values"] = self.config.transforms.std + # Add new IR keys to parameters + for key, value in self._metadata_in_ir_format().items(): + # since the same key is used to store label info in OTX SDK format + if key[1] == "labels": + assert isinstance(value, str) + configuration["modelapi_labels"] = [name for name in value.split(" ")] + elif key[1] in ("mean_values", "scale_values"): + assert isinstance(value, str) + configuration[key[1]] = [float(x) for x in value.split(" ")] + else: + configuration[key[1]] = value return configuration @@ -446,7 +521,7 @@ def deploy(self, output_model: ModelEntity) -> None: task_type = str(self.task_type).lower() - parameters["type_of_model"] = task_type + parameters["type_of_model"] = "AnomalyDetection" parameters["converter_type"] = task_type.upper() parameters["model_parameters"] = self._get_openvino_configuration() zip_buffer = io.BytesIO() @@ -455,17 +530,6 @@ def deploy(self, output_model: ModelEntity) -> None: arch.writestr(os.path.join("model", "model.xml"), self.task_environment.model.get_data("openvino.xml")) arch.writestr(os.path.join("model", "model.bin"), self.task_environment.model.get_data("openvino.bin")) arch.writestr(os.path.join("model", "config.json"), json.dumps(parameters, ensure_ascii=False, indent=4)) - # model_wrappers files - for root, _, files in os.walk( - os.path.dirname(otx.algorithms.anomaly.adapters.anomalib.exportable_code.__file__) - ): - if "__pycache__" in root: - continue - for file in files: - file_path = os.path.join(root, file) - arch.write( - file_path, os.path.join("python", "model_wrappers", file_path.split("exportable_code/")[1]) - ) # other python files arch.write(os.path.join(work_dir, "requirements.txt"), os.path.join("python", "requirements.txt")) arch.write(os.path.join(work_dir, "LICENSE"), os.path.join("python", "LICENSE")) diff --git a/src/otx/algorithms/anomaly/tasks/train.py b/src/otx/algorithms/anomaly/tasks/train.py index 639d22c65a6..a1f4759ab1a 100644 --- a/src/otx/algorithms/anomaly/tasks/train.py +++ b/src/otx/algorithms/anomaly/tasks/train.py @@ -67,7 +67,7 @@ def train( if seed: logger.info(f"Setting seed to {seed}") seed_everything(seed, workers=True) - config.trainer.deterministic = deterministic + config.trainer.deterministic = "warn" if deterministic else deterministic logger.info("Training Configs '%s'", config) diff --git a/src/otx/algorithms/common/utils/ir.py b/src/otx/algorithms/common/utils/ir.py index 6bf54131b18..946ef9b40c0 100644 --- a/src/otx/algorithms/common/utils/ir.py +++ b/src/otx/algorithms/common/utils/ir.py @@ -19,7 +19,7 @@ def check_if_quantized(model: Any) -> bool: return False -def embed_ir_model_data(xml_file: str, data_items: Dict[Tuple[str], Any]) -> None: +def embed_ir_model_data(xml_file: str, data_items: Dict[Tuple[str, str], Any]) -> None: """Embeds serialized data to IR xml file. Args: @@ -34,6 +34,6 @@ def embed_ir_model_data(xml_file: str, data_items: Dict[Tuple[str], Any]) -> Non # workaround for CVS-110054 tmp_xml_path = Path(Path(xml_file).parent) / "tmp.xml" - serialize(model, tmp_xml_path) + serialize(model, str(tmp_xml_path)) tmp_xml_path.rename(xml_file) Path(str(tmp_xml_path.parent / tmp_xml_path.stem) + ".bin").unlink() diff --git a/src/otx/algorithms/common/utils/utils.py b/src/otx/algorithms/common/utils/utils.py index 24af86d48b9..cd8bcd653b5 100644 --- a/src/otx/algorithms/common/utils/utils.py +++ b/src/otx/algorithms/common/utils/utils.py @@ -21,9 +21,10 @@ import sys from collections import defaultdict from pathlib import Path -from typing import Callable, Optional, Tuple +from typing import Any, Callable, Dict, Optional, Tuple import numpy as np +import onnx import yaml from addict import Dict as adict @@ -153,3 +154,16 @@ def read_py_config(filename: str) -> adict: ) return cfg_dict + + +def embed_onnx_model_data(onnx_file: str, extra_model_data: Dict[Tuple[str, str], Any]) -> None: + """Embeds model api config to onnx file.""" + model = onnx.load(onnx_file) + + for item in extra_model_data: + meta = model.metadata_props.add() + attr_path = " ".join(map(str, item)) + meta.key = attr_path.strip() + meta.value = str(extra_model_data[item]) + + onnx.save(model, onnx_file) diff --git a/src/otx/algorithms/visual_prompting/configs/base/configuration.py b/src/otx/algorithms/visual_prompting/configs/base/configuration.py index 63dc1e726a2..d9cdae0eaeb 100644 --- a/src/otx/algorithms/visual_prompting/configs/base/configuration.py +++ b/src/otx/algorithms/visual_prompting/configs/base/configuration.py @@ -83,17 +83,17 @@ class __Postprocessing(ParameterGroup): affects_outcome_of=ModelLifecycle.INFERENCE, ) - orig_width = configurable_float( + orig_width = configurable_integer( header="Original width", description="Model input width before embedding processing.", - default_value=64.0, + default_value=64, affects_outcome_of=ModelLifecycle.INFERENCE, ) - orig_height = configurable_float( + orig_height = configurable_integer( header="Original height", description="Model input height before embedding processing.", - default_value=64.0, + default_value=64, affects_outcome_of=ModelLifecycle.INFERENCE, ) diff --git a/src/otx/api/usecases/exportable_code/demo/demo_package/model_container.py b/src/otx/api/usecases/exportable_code/demo/demo_package/model_container.py index eda80faab7a..88725a07df3 100644 --- a/src/otx/api/usecases/exportable_code/demo/demo_package/model_container.py +++ b/src/otx/api/usecases/exportable_code/demo/demo_package/model_container.py @@ -15,8 +15,8 @@ from otx.api.entities.label_schema import LabelSchemaEntity from otx.api.entities.model_template import TaskType from otx.api.serialization.label_mapper import LabelSchemaMapper -from otx.api.utils.tiler import Tiler from otx.api.utils.detection_utils import detection2array +from otx.api.utils.tiler import Tiler from .utils import get_model_path, get_parameters @@ -49,7 +49,21 @@ def __init__(self, model_dir: Path, device="CPU") -> None: # labels for modelAPI wrappers can be empty, because unused in pre- and postprocessing self.model_parameters = self.parameters["model_parameters"] - self.model_parameters["labels"] = [] + + if self._task_type in ( + TaskType.ANOMALY_CLASSIFICATION, + TaskType.ANOMALY_DETECTION, + TaskType.ANOMALY_SEGMENTATION, + ): + # The anomaly task requires non-empty labels. + # modelapi_labels key is used as a workaround as labels key is used for labels in OTX SDK format + self.model_parameters["labels"] = ( + self.model_parameters.pop("modelapi_labels") + if "modelapi_labels" in self.model_parameters + else ["Normal", "Anomaly"] + ) + else: + self.model_parameters["labels"] = [] self._initialize_wrapper() self.core_model = Model.create_model( diff --git a/src/otx/api/usecases/exportable_code/demo/requirements.txt b/src/otx/api/usecases/exportable_code/demo/requirements.txt index d0eecdea0e1..6eda94d0e5e 100644 --- a/src/otx/api/usecases/exportable_code/demo/requirements.txt +++ b/src/otx/api/usecases/exportable_code/demo/requirements.txt @@ -1,4 +1,4 @@ openvino==2023.0 -openvino-model-api==0.1.3 +openvino-model-api==0.1.6 otx==1.4.3 numpy>=1.21.0,<=1.23.5 # np.bool was removed in 1.24.0 which was used in openvino runtime diff --git a/src/otx/api/usecases/exportable_code/prediction_to_annotation_converter.py b/src/otx/api/usecases/exportable_code/prediction_to_annotation_converter.py index b9931aed01e..90ba92a0c4e 100644 --- a/src/otx/api/usecases/exportable_code/prediction_to_annotation_converter.py +++ b/src/otx/api/usecases/exportable_code/prediction_to_annotation_converter.py @@ -10,6 +10,7 @@ import cv2 import numpy as np from openvino.model_api.models import utils +from openvino.model_api.models.utils import AnomalyResult from otx.api.entities.annotation import ( Annotation, @@ -20,10 +21,9 @@ from otx.api.entities.label import Domain from otx.api.entities.label_schema import LabelSchemaEntity from otx.api.entities.scored_label import ScoredLabel -from otx.api.entities.shapes.polygon import Point, Polygon from otx.api.entities.shapes.ellipse import Ellipse +from otx.api.entities.shapes.polygon import Point, Polygon from otx.api.entities.shapes.rectangle import Rectangle -from otx.api.utils.anomaly_utils import create_detection_annotation_from_anomaly_heatmap from otx.api.utils.labels_utils import get_empty_label from otx.api.utils.segmentation_utils import create_annotation_from_segmentation_map from otx.api.utils.time_utils import now @@ -321,7 +321,7 @@ def __init__(self, label_schema: LabelSchemaEntity): self.normal_label = [label for label in labels if not label.is_anomalous][0] self.anomalous_label = [label for label in labels if label.is_anomalous][0] - def convert_to_annotation(self, predictions: np.ndarray, metadata: Dict[str, Any]) -> AnnotationSceneEntity: + def convert_to_annotation(self, predictions: AnomalyResult, metadata: Dict[str, Any]) -> AnnotationSceneEntity: """Convert predictions to OTX Annotation Scene using the metadata. Args: @@ -331,15 +331,14 @@ def convert_to_annotation(self, predictions: np.ndarray, metadata: Dict[str, Any Returns: AnnotationSceneEntity: OTX annotation scene entity object. """ - pred_label = predictions >= metadata.get("threshold", 0.5) - - label = self.anomalous_label if pred_label else self.normal_label - probability = (1 - predictions) if predictions < 0.5 else predictions + assert predictions.pred_score is not None + assert predictions.pred_label is not None + label = self.anomalous_label if predictions.pred_label == "Anomaly" else self.normal_label annotations = [ Annotation( Rectangle.generate_full_box(), - labels=[ScoredLabel(label=label, probability=float(probability))], + labels=[ScoredLabel(label=label, probability=float(predictions.pred_score))], ) ] return AnnotationSceneEntity(kind=AnnotationSceneKind.PREDICTION, annotations=annotations) @@ -358,19 +357,21 @@ def __init__(self, label_schema: LabelSchemaEntity): self.anomalous_label = [label for label in labels if label.is_anomalous][0] self.label_map = {0: self.normal_label, 1: self.anomalous_label} - def convert_to_annotation(self, predictions: np.ndarray, metadata: Dict[str, Any]) -> AnnotationSceneEntity: + def convert_to_annotation(self, predictions: AnomalyResult, metadata: Dict[str, Any]) -> AnnotationSceneEntity: """Convert predictions to OTX Annotation Scene using the metadata. Args: - predictions (tuple): Raw predictions from the model. + predictions (AnomalyResult): Raw predictions from the model. metadata (Dict[str, Any]): Variable containing metadata information. Returns: AnnotationSceneEntity: OTX annotation scene entity object. """ - pred_mask = predictions >= 0.5 - mask = pred_mask.squeeze().astype(np.uint8) - annotations = create_annotation_from_segmentation_map(mask, predictions, self.label_map) + assert predictions.pred_mask is not None + assert predictions.anomaly_map is not None + annotations = create_annotation_from_segmentation_map( + predictions.pred_mask, predictions.anomaly_map, self.label_map + ) if len(annotations) == 0: # TODO: add confidence to this label annotations = [ @@ -400,7 +401,7 @@ def __init__(self, label_schema: LabelSchemaEntity): self.anomalous_label = [label for label in labels if label.is_anomalous][0] self.label_map = {0: self.normal_label, 1: self.anomalous_label} - def convert_to_annotation(self, predictions: np.ndarray, metadata: Dict[str, Any]) -> AnnotationSceneEntity: + def convert_to_annotation(self, predictions: AnomalyResult, metadata: Dict[str, Any]) -> AnnotationSceneEntity: """Convert predictions to OTX Annotation Scene using the metadata. Args: @@ -410,9 +411,18 @@ def convert_to_annotation(self, predictions: np.ndarray, metadata: Dict[str, Any Returns: AnnotationSceneEntity: OTX annotation scene entity object. """ - pred_mask = predictions >= 0.5 - mask = pred_mask.squeeze().astype(np.uint8) - annotations = create_detection_annotation_from_anomaly_heatmap(mask, predictions, self.label_map) + assert predictions.pred_boxes is not None + assert predictions.pred_score is not None + assert predictions.pred_mask is not None + annotations = [] + image_h, image_w = predictions.pred_mask.shape + for box in predictions.pred_boxes: + annotations.append( + Annotation( + Rectangle(box[0] / image_w, box[1] / image_h, box[2] / image_w, box[3] / image_h), + labels=[ScoredLabel(label=self.anomalous_label, probability=predictions.pred_score)], + ) + ) if len(annotations) == 0: # TODO: add confidence to this label annotations = [ diff --git a/tests/e2e/cli/anomaly/reference/ote_anomaly_classification_padim/compressed_model.yml b/tests/e2e/cli/anomaly/reference/ote_anomaly_classification_padim/compressed_model.yml index 01460cc560c..1c94650f275 100644 --- a/tests/e2e/cli/anomaly/reference/ote_anomaly_classification_padim/compressed_model.yml +++ b/tests/e2e/cli/anomaly/reference/ote_anomaly_classification_padim/compressed_model.yml @@ -1,5 +1,7 @@ TestToolsAnomalyClassification: nncf: number_of_fakequantizers: 27 + ptq: + number_of_fakequantizers: 28 pot: number_of_fakequantizers: 28 diff --git a/tests/e2e/cli/anomaly/reference/ote_anomaly_detection_padim/compressed_model.yml b/tests/e2e/cli/anomaly/reference/ote_anomaly_detection_padim/compressed_model.yml index 4bca1f02a5d..c119a9d5f52 100644 --- a/tests/e2e/cli/anomaly/reference/ote_anomaly_detection_padim/compressed_model.yml +++ b/tests/e2e/cli/anomaly/reference/ote_anomaly_detection_padim/compressed_model.yml @@ -1,5 +1,7 @@ TestToolsAnomalyDetection: nncf: number_of_fakequantizers: 27 + ptq: + number_of_fakequantizers: 28 pot: number_of_fakequantizers: 28 diff --git a/tests/e2e/cli/anomaly/reference/ote_anomaly_segmentation_padim/compressed_model.yml b/tests/e2e/cli/anomaly/reference/ote_anomaly_segmentation_padim/compressed_model.yml index 1d41886dfd4..217f636a463 100644 --- a/tests/e2e/cli/anomaly/reference/ote_anomaly_segmentation_padim/compressed_model.yml +++ b/tests/e2e/cli/anomaly/reference/ote_anomaly_segmentation_padim/compressed_model.yml @@ -1,5 +1,7 @@ TestToolsAnomalySegmentation: nncf: number_of_fakequantizers: 27 + ptq: + number_of_fakequantizers: 28 pot: number_of_fakequantizers: 28 diff --git a/tests/e2e/cli/anomaly/test_anomaly_classification.py b/tests/e2e/cli/anomaly/test_anomaly_classification.py index 7dade9a11c6..10da1239ea4 100644 --- a/tests/e2e/cli/anomaly/test_anomaly_classification.py +++ b/tests/e2e/cli/anomaly/test_anomaly_classification.py @@ -58,12 +58,12 @@ class TestToolsAnomalyClassification: @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_otx_train(self, template, tmp_dir_path): - otx_train_testing(template, tmp_dir_path, otx_dir, args, deterministic=False) + otx_train_testing(template, tmp_dir_path, otx_dir, args, deterministic=True) @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_otx_export(self, template, tmp_dir_path): - otx_export_testing(template, tmp_dir_path) + otx_export_testing(template, tmp_dir_path, check_ir_meta=True) @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) diff --git a/tests/e2e/cli/anomaly/test_anomaly_detection.py b/tests/e2e/cli/anomaly/test_anomaly_detection.py index 9068da92a10..10b008980ac 100644 --- a/tests/e2e/cli/anomaly/test_anomaly_detection.py +++ b/tests/e2e/cli/anomaly/test_anomaly_detection.py @@ -58,12 +58,12 @@ class TestToolsAnomalyDetection: @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_otx_train(self, template, tmp_dir_path): - otx_train_testing(template, tmp_dir_path, otx_dir, args, deterministic=False) + otx_train_testing(template, tmp_dir_path, otx_dir, args, deterministic=True) @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_otx_export(self, template, tmp_dir_path): - otx_export_testing(template, tmp_dir_path) + otx_export_testing(template, tmp_dir_path, check_ir_meta=True) @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) diff --git a/tests/e2e/cli/anomaly/test_anomaly_segmentation.py b/tests/e2e/cli/anomaly/test_anomaly_segmentation.py index 50e9d0da2a2..a3863debf25 100644 --- a/tests/e2e/cli/anomaly/test_anomaly_segmentation.py +++ b/tests/e2e/cli/anomaly/test_anomaly_segmentation.py @@ -58,12 +58,12 @@ class TestToolsAnomalySegmentation: @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_otx_train(self, template, tmp_dir_path): - otx_train_testing(template, tmp_dir_path, otx_dir, args, deterministic=False) + otx_train_testing(template, tmp_dir_path, otx_dir, args, deterministic=True) @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_otx_export(self, template, tmp_dir_path): - otx_export_testing(template, tmp_dir_path) + otx_export_testing(template, tmp_dir_path, check_ir_meta=True) @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) diff --git a/tests/integration/cli/anomaly/test_anomaly_classification.py b/tests/integration/cli/anomaly/test_anomaly_classification.py index c9173e08128..72ae511c8a9 100644 --- a/tests/integration/cli/anomaly/test_anomaly_classification.py +++ b/tests/integration/cli/anomaly/test_anomaly_classification.py @@ -11,6 +11,7 @@ from otx.cli.registry import Registry from tests.test_suite.e2e_test_system import e2e_pytest_component from tests.test_suite.run_test_command import ( + generate_model_template_testing, nncf_optimize_testing, otx_deploy_openvino_testing, otx_eval_deployment_testing, @@ -18,7 +19,6 @@ otx_eval_testing, otx_export_testing, otx_train_testing, - generate_model_template_testing, ) args = { @@ -42,7 +42,7 @@ class TestToolsAnomalyClassification: @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_otx_train(self, template, tmp_dir_path): - otx_train_testing(template, tmp_dir_path, otx_dir, args, deterministic=False) + otx_train_testing(template, tmp_dir_path, otx_dir, args, deterministic=True) @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) @@ -57,7 +57,7 @@ def test_otx_export_fp16(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_otx_export_onnx(self, template, tmp_dir_path): - otx_export_testing(template, tmp_dir_path, half_precision=False, is_onnx=True) + otx_export_testing(template, tmp_dir_path, half_precision=False, is_onnx=True, check_ir_meta=True) @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) diff --git a/tests/integration/cli/anomaly/test_anomaly_detection.py b/tests/integration/cli/anomaly/test_anomaly_detection.py index 0cb33a51fd8..795c64ab49d 100644 --- a/tests/integration/cli/anomaly/test_anomaly_detection.py +++ b/tests/integration/cli/anomaly/test_anomaly_detection.py @@ -11,6 +11,7 @@ from otx.cli.registry import Registry from tests.test_suite.e2e_test_system import e2e_pytest_component from tests.test_suite.run_test_command import ( + generate_model_template_testing, nncf_optimize_testing, otx_deploy_openvino_testing, otx_eval_deployment_testing, @@ -18,7 +19,6 @@ otx_eval_testing, otx_export_testing, otx_train_testing, - generate_model_template_testing, ) args = { @@ -42,7 +42,7 @@ class TestToolsAnomalyDetection: @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_otx_train(self, template, tmp_dir_path): - otx_train_testing(template, tmp_dir_path, otx_dir, args, deterministic=False) + otx_train_testing(template, tmp_dir_path, otx_dir, args, deterministic=True) @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) @@ -57,7 +57,7 @@ def test_otx_export_fp16(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_otx_export_onnx(self, template, tmp_dir_path): - otx_export_testing(template, tmp_dir_path, half_precision=False, is_onnx=True) + otx_export_testing(template, tmp_dir_path, half_precision=False, is_onnx=True, check_ir_meta=True) @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) diff --git a/tests/integration/cli/anomaly/test_anomaly_segmentation.py b/tests/integration/cli/anomaly/test_anomaly_segmentation.py index 17483504515..43366a80676 100644 --- a/tests/integration/cli/anomaly/test_anomaly_segmentation.py +++ b/tests/integration/cli/anomaly/test_anomaly_segmentation.py @@ -11,6 +11,7 @@ from otx.cli.registry import Registry from tests.test_suite.e2e_test_system import e2e_pytest_component from tests.test_suite.run_test_command import ( + generate_model_template_testing, nncf_optimize_testing, otx_deploy_openvino_testing, otx_eval_deployment_testing, @@ -18,7 +19,6 @@ otx_eval_testing, otx_export_testing, otx_train_testing, - generate_model_template_testing, ) args = { @@ -42,7 +42,7 @@ class TestToolsAnomalySegmentation: @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_otx_train(self, template, tmp_dir_path): - otx_train_testing(template, tmp_dir_path, otx_dir, args, deterministic=False) + otx_train_testing(template, tmp_dir_path, otx_dir, args, deterministic=True) @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) @@ -57,7 +57,7 @@ def test_otx_export_fp16(self, template, tmp_dir_path): @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) def test_otx_export_onnx(self, template, tmp_dir_path): - otx_export_testing(template, tmp_dir_path, half_precision=False, is_onnx=True) + otx_export_testing(template, tmp_dir_path, half_precision=False, is_onnx=True, check_ir_meta=True) @e2e_pytest_component @pytest.mark.parametrize("template", templates, ids=templates_ids) diff --git a/tests/unit/algorithms/anomaly/tasks/test_openvino.py b/tests/unit/algorithms/anomaly/tasks/test_openvino.py index 82d1174bb97..8fb222189aa 100644 --- a/tests/unit/algorithms/anomaly/tasks/test_openvino.py +++ b/tests/unit/algorithms/anomaly/tasks/test_openvino.py @@ -91,7 +91,7 @@ def test_openvino(self, tmpdir, setup_task_environment): openvino_task.deploy(output_model) assert output_model.exportable_code is not None - @patch.multiple(OpenVINOTask, get_config=MagicMock(), load_inferencer=MagicMock()) + @patch.multiple(OpenVINOTask, get_config=MagicMock(), get_openvino_model=MagicMock()) @patch("otx.algorithms.anomaly.tasks.openvino.get_transforms", MagicMock()) def test_anomaly_legacy_keys(self, mocker, tmp_dir): """Checks whether the model is loaded correctly with legacy and current keys.""" diff --git a/tests/unit/api/usecases/exportable_code/test_prediction_to_annotation_converter.py b/tests/unit/api/usecases/exportable_code/test_prediction_to_annotation_converter.py index f88847fbe9c..fdc81c94101 100644 --- a/tests/unit/api/usecases/exportable_code/test_prediction_to_annotation_converter.py +++ b/tests/unit/api/usecases/exportable_code/test_prediction_to_annotation_converter.py @@ -933,69 +933,3 @@ def test_anomaly_classification_to_annotation_init( converter = AnomalyClassificationToAnnotationConverter(label_schema=label_schema) assert converter.normal_label == non_empty_labels[0] assert converter.anomalous_label == non_empty_labels[2] - - @pytest.mark.priority_medium - @pytest.mark.unit - @pytest.mark.reqids(Requirements.REQ_1) - def test_anomaly_classification_to_annotation_convert( - self, - ): - """ - Description: - Check "AnomalyClassificationToAnnotationConverter" class "convert_to_annotation" method - - Input data: - "AnomalyClassificationToAnnotationConverter" class object, "predictions" array - - Expected results: - Test passes if "AnnotationSceneEntity" object returned by "convert_to_annotation" method is equal to - expected - - Steps - 1. Check attributes of "AnnotationSceneEntity" object returned by "convert_to_annotation" method for - "metadata" dictionary with specified "threshold" key - 2. Check attributes of "AnnotationSceneEntity" object returned by "convert_to_annotation" method for - "metadata" dictionary without specified "threshold" key - """ - - def check_annotation(actual_annotation: Annotation, expected_labels: list): - assert isinstance(actual_annotation, Annotation) - assert actual_annotation.get_labels() == expected_labels - assert isinstance(actual_annotation.shape, Rectangle) - assert Rectangle.is_full_box(rectangle=actual_annotation.shape) - - non_empty_labels = [ - LabelEntity(name="Normal", domain=Domain.CLASSIFICATION, id=ID("1")), - LabelEntity( - name="Anomalous", - domain=Domain.CLASSIFICATION, - id=ID("2"), - is_anomalous=True, - ), - ] - label_group = LabelGroup(name="Anomaly classification labels group", labels=non_empty_labels) - label_schema = LabelSchemaEntity(label_groups=[label_group]) - converter = AnomalyClassificationToAnnotationConverter(label_schema=label_schema) - predictions = np.array([0.7]) - # Checking attributes of "AnnotationSceneEntity" returned by "convert_to_annotation" for "metadata" with - # specified "threshold" key - metadata = { - "non-required key": 1, - "other non-required key": 2, - "threshold": 0.8, - } - predictions_to_annotations = converter.convert_to_annotation(predictions=predictions, metadata=metadata) - check_annotation_scene(annotation_scene=predictions_to_annotations, expected_length=1) - check_annotation( - predictions_to_annotations.annotations[0], - expected_labels=[ScoredLabel(label=non_empty_labels[0], probability=0.7)], - ) - # Checking attributes of "AnnotationSceneEntity" returned by "convert_to_annotation" for "metadata" without - # specified "threshold" key - metadata = {"non-required key": 1, "other non-required key": 2} - predictions_to_annotations = converter.convert_to_annotation(predictions=predictions, metadata=metadata) - check_annotation_scene(annotation_scene=predictions_to_annotations, expected_length=1) - check_annotation( - predictions_to_annotations.annotations[0], - expected_labels=[ScoredLabel(label=non_empty_labels[1], probability=0.7)], - )