v0.3.1

Signed-off-by: Kory Watson <[email protected]>

mypy.ini

@@ -74,5 +74,11 @@ ignore_missing_imports = True
 [mypy-ruamel.*]
 ignore_missing_imports = True
 
+[mypy-pandas.*]
+ignore_missing_imports = True
+
+[mypy-tabulate.*]
+ignore_missing_imports = True
+
 [mypy-qai_hub_models.models.*]
 ignore_errors = true

qai_hub_models/_version.py

@@ -2,4 +2,4 @@
 # Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved.
 # SPDX-License-Identifier: BSD-3-Clause
 # ---------------------------------------------------------------------
-__version__ = "0.3.0"
+__version__ = "0.3.1"

qai_hub_models/datasets/imagenette.py

@@ -8,7 +8,6 @@
 from torchvision.datasets import ImageNet
 
 from qai_hub_models.datasets.common import BaseDataset
-from qai_hub_models.models._shared.imagenet_classifier.app import IMAGENET_TRANSFORM
 from qai_hub_models.utils.asset_loaders import CachedWebDatasetAsset
 
 IMAGENETTE_FOLDER_NAME = "imagenette2-320"
@@ -53,6 +52,11 @@ class ImagenetteDataset(BaseDataset, ImageNet):
     def __init__(self):
         self._download_data()
         BaseDataset.__init__(self, str(IMAGENETTE_ASSET.path(extracted=True)))
+        # Avoid circular import
+        from qai_hub_models.models._shared.imagenet_classifier.app import (
+            IMAGENET_TRANSFORM,
+        )
+
         ImageNet.__init__(
             self,
             root=IMAGENETTE_ASSET.path(),
@@ -94,7 +98,6 @@ def _download_data(self) -> None:
         devkit_path = DEVKIT_ASSET.fetch()
         devkit_st = os.stat(devkit_path)
         os.chmod(devkit_path, devkit_st.st_mode | stat.S_IEXEC)
-        os.symlink(
-            DEVKIT_ASSET.path(),
-            IMAGENETTE_ASSET.path() / os.path.basename(DEVKIT_ASSET.path()),
-        )
+        target_path = IMAGENETTE_ASSET.path() / os.path.basename(DEVKIT_ASSET.path())
+        if not os.path.exists(target_path):
+            os.symlink(DEVKIT_ASSET.path(), target_path)

qai_hub_models/models/_shared/cityscapes_segmentation/demo.py

@@ -14,13 +14,13 @@
     CityscapesSegmentor,
 )
 from qai_hub_models.utils.args import (
-    TargetRuntime,
     demo_model_from_cli_args,
     get_model_cli_parser,
     get_on_device_demo_parser,
     validate_on_device_demo_args,
 )
 from qai_hub_models.utils.asset_loaders import CachedWebModelAsset, load_image
+from qai_hub_models.utils.base_model import TargetRuntime
 from qai_hub_models.utils.display import display_or_save_image
 from qai_hub_models.utils.image_processing import pil_resize_pad, pil_undo_resize_pad
 
@@ -65,13 +65,15 @@ def cityscapes_segmentation_demo(
 
     (_, _, height, width) = input_spec["image"][0]
     orig_image = load_image(image)
-    image, _, padding = pil_resize_pad(orig_image, (height, width))
+    image, scale, padding = pil_resize_pad(orig_image, (height, width))
 
     # Run app
     image_annotated = app.predict(image)
 
     # Resize / unpad annotated image
-    image_annotated = pil_undo_resize_pad(image_annotated, orig_image.size, padding)
+    image_annotated = pil_undo_resize_pad(
+        image_annotated, orig_image.size, scale, padding
+    )
 
     if not is_test:
         display_or_save_image(

qai_hub_models/models/_shared/deeplab/demo.py

@@ -8,13 +8,15 @@
 
 from qai_hub_models.models._shared.deeplab.app import DeepLabV3App
 from qai_hub_models.utils.args import (
-    add_output_dir_arg,
+    demo_model_from_cli_args,
     get_model_cli_parser,
-    model_from_cli_args,
+    get_on_device_demo_parser,
+    validate_on_device_demo_args,
 )
 from qai_hub_models.utils.asset_loaders import CachedWebAsset, load_image
 from qai_hub_models.utils.base_model import BaseModel
 from qai_hub_models.utils.display import display_or_save_image
+from qai_hub_models.utils.image_processing import pil_resize_pad, pil_undo_resize_pad
 
 
 def deeplabv3_demo(
@@ -25,21 +27,38 @@ def deeplabv3_demo(
 ):
     # Demo parameters
     parser = get_model_cli_parser(model_type)
+    parser = get_on_device_demo_parser(parser, add_output_dir=True)
     parser.add_argument(
         "--image",
         type=str,
         default=default_image,
         help="image file path or URL.",
     )
-    add_output_dir_arg(parser)
     args = parser.parse_args([] if is_test else None)
+    validate_on_device_demo_args(args, model_type.get_model_id())
+
+    input_spec = model_type.get_input_spec()
+
+    # load image and model
+    (_, _, height, width) = input_spec["image"][0]
+    orig_image = load_image(args.image)
+    image, scale, padding = pil_resize_pad(orig_image, (height, width))
 
     # This DeepLabV3 ResNet 50 demo comes from
     # https://pytorch.org/hub/pytorch_vision_deeplabv3_resnet101/
-    # load image and model
-    image = load_image(args.image)
     input_image = image.convert("RGB")
-    app = DeepLabV3App(model_from_cli_args(model_type, args), num_classes=num_classes)
-    output = app.predict(input_image, False)
+    inference_model = demo_model_from_cli_args(model_type, args)
+    app = DeepLabV3App(inference_model, num_classes=num_classes)
+
+    # Run app
+    image_annotated = app.predict(input_image, False)
+
+    # Resize / unpad annotated image
+    image_annotated = pil_undo_resize_pad(
+        image_annotated, orig_image.size, scale, padding
+    )
+
     if not is_test:
-        display_or_save_image(output, args.output_dir)
+        display_or_save_image(
+            image_annotated, args.output_dir, "annotated_image", "predicted image"
+        )

qai_hub_models/models/_shared/ffnet_quantized/model.py

@@ -4,6 +4,14 @@
 # ---------------------------------------------------------------------
 from __future__ import annotations
 
+# isort: off
+# This verifies aimet is installed, and this must be included first.
+from qai_hub_models.utils.quantization_aimet import (
+    AIMETQuantizableMixin,
+)
+
+# isort: on
+
 import os
 
 import torch
@@ -15,7 +23,6 @@
 from qai_hub_models.models._shared.ffnet.model import FFNet
 from qai_hub_models.utils.base_model import SourceModelFormat, TargetRuntime
 from qai_hub_models.utils.input_spec import InputSpec
-from qai_hub_models.utils.quantization_aimet import AIMETQuantizableMixin
 
 MODEL_ID = __name__.split(".")[-2]
 FFNET_AIMET_CONFIG = os.path.abspath(

qai_hub_models/models/_shared/imagenet_classifier/model.py

@@ -4,10 +4,12 @@
 # ---------------------------------------------------------------------
 from __future__ import annotations
 
-from typing import Optional
+from typing import Dict, List, Optional
 
+import numpy as np
 import torch
 
+from qai_hub_models.datasets.imagenette import ImagenetteDataset
 from qai_hub_models.evaluators.base_evaluators import BaseEvaluator
 from qai_hub_models.evaluators.classification_evaluator import ClassificationEvaluator
 from qai_hub_models.utils.base_model import BaseModel
@@ -73,3 +75,9 @@ def from_pretrained(
     ) -> "ImagenetClassifier":
         net = cls.model_builder(weights=weights or cls.DEFAULT_WEIGHTS)
         return cls(net)
+
+    def sample_inputs(
+        self, input_spec: InputSpec | None = None
+    ) -> Dict[str, List[np.ndarray]]:
+        dataset = ImagenetteDataset()
+        return dict(image_tensor=[dataset[42][0].numpy()[None, :, :, :]])

qai_hub_models/models/baichuan_7b_quantized/README.md

@@ -16,12 +16,16 @@ a hosted Qualcomm® device.
 
 
 ## License
-- Code in the Qualcomm® AI Hub Models repository is covered by the LICENSE
-  file at the repository root.
 - The license for the original implementation of Baichuan-7B can be found
   [here](https://github.com/baichuan-inc/Baichuan-7B/blob/main/LICENSE).
-
+- The license for the compiled assets for on-device deployment can be found [here](https://qaihub-public-assets.s3.us-west-2.amazonaws.com/qai-hub-models/Qualcomm+AI+Hub+Proprietary+License.pdf).
 
 ## References
 * [Baichuan 2: Open Large-scale Language Models](https://arxiv.org/abs/2309.10305)
 * [Source Model Implementation](https://github.com/baichuan-inc/Baichuan-7B/)
+
+## Community
+* Join [our AI Hub Slack community](https://join.slack.com/t/qualcommaihub-nac3926/shared_invite/zt-2d5zsmas3-Sj0Q9TzslueCjS31eXG2UA) to collaborate, post questions and learn more about on-device AI.
+* For questions or feedback please [reach out to us](mailto:[email protected]).
+
+{usage_and_limitation}

qai_hub_models/models/controlnet_quantized/README.md

@@ -44,12 +44,16 @@ Additional options are documented with the `--help` option. Note that the above
 script requires access to Deployment instructions for Qualcomm® AI Hub.
 
 ## License
-- Code in the Qualcomm® AI Hub Models repository is covered by the LICENSE
-  file at the repository root.
 - The license for the original implementation of ControlNet can be found
   [here](https://github.com/lllyasviel/ControlNet/blob/main/LICENSE).
-
+- The license for the compiled assets for on-device deployment can be found [here](https://qaihub-public-assets.s3.us-west-2.amazonaws.com/qai-hub-models/Qualcomm+AI+Hub+Proprietary+License.pdf).
 
 ## References
 * [Adding Conditional Control to Text-to-Image Diffusion Models](https://arxiv.org/abs/2302.05543)
 * [Source Model Implementation](https://github.com/lllyasviel/ControlNet)
+
+## Community
+* Join [our AI Hub Slack community](https://join.slack.com/t/qualcommaihub-nac3926/shared_invite/zt-2d5zsmas3-Sj0Q9TzslueCjS31eXG2UA) to collaborate, post questions and learn more about on-device AI.
+* For questions or feedback please [reach out to us](mailto:[email protected]).
+
+{usage_and_limitation}

qai_hub_models/models/controlnet_quantized/demo.py

@@ -23,27 +23,39 @@
 from qai_hub_models.utils.asset_loaders import CachedWebModelAsset, load_image
 from qai_hub_models.utils.base_model import BasePrecompiledModel
 from qai_hub_models.utils.display import display_or_save_image
-from qai_hub_models.utils.inference import HubModel
+from qai_hub_models.utils.inference import HubModel, get_uploaded_precompiled_model
 from qai_hub_models.utils.qai_hub_helpers import can_access_qualcomm_ai_hub
 
 INPUT_IMAGE = CachedWebModelAsset.from_asset_store(
     MODEL_ID, MODEL_ASSET_VERSION, "test_images/test_bird_image.png"
 ).fetch()
 
 DEFAULT_DEMO_PROMPT = "a white bird on a colorful window"
+DEFAULT_DEVICE_NAME = "Samsung Galaxy S23 Ultra"
 
 
-def _get_hub_model(input_model: BasePrecompiledModel, device=DEFAULT_EXPORT_DEVICE):
+def _get_hub_model(
+    input_model: BasePrecompiledModel,
+    model_name: str,
+    ignore_cached_model: bool = False,
+    device_name=DEFAULT_DEVICE_NAME,
+):
     if not can_access_qualcomm_ai_hub():
         raise RuntimeError(
             "ControlNet on-device demo requires access to QAI-Hub.\n"
             "Please visit https://aihub.qualcomm.com/ and sign-up."
         )
 
     # Upload model
-    uploaded_model = hub.upload_model(input_model.get_target_model_path())
+    uploaded_model = get_uploaded_precompiled_model(
+        input_model.get_target_model_path(),
+        MODEL_ID,
+        MODEL_ASSET_VERSION,
+        model_name,
+        ignore_cached_model=ignore_cached_model,
+    )
     inputs = list(input_model.get_input_spec().keys())
-    return HubModel(uploaded_model, inputs, hub.Device(name=device))
+    return HubModel(uploaded_model, inputs, hub.Device(name=device_name))
 
 
 # Run ControlNet end-to-end on a given prompt and input image.
@@ -82,7 +94,12 @@ def main(is_test: bool = False):
         help="Strength of guidance (higher means more influence from prompt).",
     )
     parser.add_argument(
-        "--device",
+        "--ignore-cached-model",
+        action="store_true",
+        help="Uploads model ignoring previously uploaded and cached model.",
+    )
+    parser.add_argument(
+        "--device-name",
         type=str,
         default=DEFAULT_EXPORT_DEVICE,
         help="Device to run stable-diffusion demo on.",
@@ -92,7 +109,7 @@ def main(is_test: bool = False):
     if not is_test:
         print(f"\n{'-' * 100}")
         print(
-            f"** Performing image generation on-device({args.device}) with ControlNet - Stable Diffusion **"
+            f"** Performing image generation on-device({args.device_name}) with ControlNet - Stable Diffusion **"
         )
         print()
         print("Prompt:", args.prompt)
@@ -102,7 +119,7 @@ def main(is_test: bool = False):
         print("Seed:", args.seed)
         print()
         print(
-            "Note: This reference demo uses significant amounts of memory and may take 5-10 minutes to run per step."
+            "Note: This reference demo uses significant amounts of memory and may take 5-10 minutes to run ** per step **."
         )
         print(f"{'-' * 100}\n")
 
@@ -118,10 +135,16 @@ def main(is_test: bool = False):
     # Instead of forward, we later submit inference_jobs on QAI-Hub for
     # on-device evaluation.
     print(f"Uploading model assets on QAI-Hub\n{'-' * 35}")
-    text_encoder = _get_hub_model(text_encoder, args.device)
-    unet = _get_hub_model(unet, args.device)
-    vae_decoder = _get_hub_model(vae_decoder, args.device)
-    controlnet = _get_hub_model(controlnet, args.device)
+    text_encoder = _get_hub_model(
+        text_encoder, "text_encoder", args.ignore_cached_model, args.device_name
+    )
+    unet = _get_hub_model(unet, "unet", args.ignore_cached_model, args.device_name)
+    vae_decoder = _get_hub_model(
+        vae_decoder, "vae_decoder", args.ignore_cached_model, args.device_name
+    )
+    controlnet = _get_hub_model(
+        controlnet, "controlnet", args.ignore_cached_model, args.device_name
+    )
 
     # Create tokenizer, scheduler and time_embedding required
     # for control-net pipeline.

qai_hub_models/models/controlnet_quantized/export.py

@@ -14,15 +14,26 @@
 import qai_hub as hub
 
 from qai_hub_models.models.controlnet_quantized import Model
-from qai_hub_models.utils.args import TargetRuntime, export_parser
+from qai_hub_models.utils.args import export_parser
+from qai_hub_models.utils.base_model import TargetRuntime
 from qai_hub_models.utils.printing import print_profile_metrics_from_job
 from qai_hub_models.utils.qai_hub_helpers import (
     can_access_qualcomm_ai_hub,
     export_without_hub_access,
 )
 
-ALL_COMPONENTS = ["text_encoder", "unet", "vae_decoder", "controlnet"]
-DEFAULT_COMPONENTS = ["text_encoder", "vae_decoder", "unet", "controlnet"]
+ALL_COMPONENTS = [
+    "Text-Encoder-Quantized",
+    "UNet-Quantized",
+    "VAE-Decoder-Quantized",
+    "ControlNet-Quantized",
+]
+DEFAULT_COMPONENTS = [
+    "Text-Encoder-Quantized",
+    "VAE-Decoder-Quantized",
+    "UNet-Quantized",
+    "ControlNet-Quantized",
+]
 
 
 def export_model(
@@ -97,14 +108,14 @@ def export_model(
     print("Initializing model class")
     model = Model.from_precompiled()
     components_dict = {}
-    if "text_encoder" in components:
-        components_dict["text_encoder"] = model.text_encoder
-    if "unet" in components:
-        components_dict["unet"] = model.unet
-    if "vae_decoder" in components:
-        components_dict["vae_decoder"] = model.vae_decoder
-    if "controlnet" in components:
-        components_dict["controlnet"] = model.controlnet
+    if "Text-Encoder-Quantized" in components:
+        components_dict["Text-Encoder-Quantized"] = model.text_encoder
+    if "UNet-Quantized" in components:
+        components_dict["UNet-Quantized"] = model.unet
+    if "VAE-Decoder-Quantized" in components:
+        components_dict["VAE-Decoder-Quantized"] = model.vae_decoder
+    if "ControlNet-Quantized" in components:
+        components_dict["ControlNet-Quantized"] = model.controlnet
 
     # 2. Upload model assets to hub
     print("Uploading model assets on hub")