v0.8.0

See https://github.com/quic/ai-hub-models/releases/v0.8.0 for changelog.

Signed-off-by: QAIHM Team <[email protected]>

qai_hub_models/_version.py

@@ -2,4 +2,4 @@
 # Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved.
 # SPDX-License-Identifier: BSD-3-Clause
 # ---------------------------------------------------------------------
-__version__ = "0.7.0"
+__version__ = "0.8.0"

qai_hub_models/asset_bases.yaml

@@ -6,6 +6,7 @@ model_asset_folder: models/{model_id}/v{version}
 dataset_asset_folder: datasets/{dataset_id}/v{version}
 repo_url: https://github.com/quic/ai-hub-models/blob/main
 qaihm_repo: qai_hub_models/models/{model_id}
+labels_path: qai_hub_models/labels/{labels_file}
 example_use: qai_hub_models/models/{model_id}#example--usage
 huggingface_path: qualcomm/{model_name}
 models_website_url: https://aihub.qualcomm.com

qai_hub_models/evaluators/classification_evaluator.py

@@ -19,20 +19,32 @@ def __init__(self, num_classes: int = 1000):
     def add_batch(self, output: torch.Tensor, gt: int | torch.Tensor):
         # This evaluator supports only 1 output tensor at a time.
         assert len(output.shape) == 2 and output.shape[-1] == self.num_classes
-        gt_tensor = torch.Tensor(gt)
-        assert len(gt_tensor.shape) == 1 and gt_tensor.shape[0] == output.shape[0]
+        gt_tensor = torch.Tensor(gt).unsqueeze(1)
+        assert len(gt_tensor.shape) == 2 and gt_tensor.shape[0] == output.shape[0]
         batch_size = output.shape[0]
         self.total_samples += batch_size
-        self.num_correct += sum(torch.argmax(output, dim=-1) == gt_tensor)
+
+        top5 = torch.topk(output, 5).indices
+        self.top5_count += torch.sum(top5 == gt_tensor).item()
+        self.top1_count += torch.sum(top5[:, :1] == gt_tensor).item()
 
     def reset(self):
-        self.num_correct = 0
+        self.top1_count = 0
+        self.top5_count = 0
         self.total_samples = 0
 
-    def get_accuracy_score(self) -> float:
+    def top1(self) -> float:
         if self.total_samples == 0:
             return 0
-        return self.num_correct / self.total_samples
+        return self.top1_count / self.total_samples
+
+    def top5(self) -> float:
+        if self.total_samples == 0:
+            return 0
+        return self.top5_count / self.total_samples
+
+    def get_accuracy_score(self) -> float:
+        return self.top1()
 
     def formatted_accuracy(self) -> str:
-        return f"{self.get_accuracy_score() * 100:.1f}%"
+        return f"{self.top1() * 100:.1f}% (Top 1), {self.top5() * 100:.1f}% (Top 5)"

qai_hub_models/global_requirements.txt

@@ -3,34 +3,45 @@
 # - Then install this requirements file
 # That should create an environment that works for every single model.
 
+# THIS FILE WAS AUTO-GENERATED. DO NOT EDIT MANUALLY.
+
 Deprecated==1.2.11
 PySoundFile; sys_platform == 'win32'
 aimet-torch==1.31.2; sys_platform == "linux"
 albumentations==0.5.2
-av==10.0.0
 basicsr==1.4.2
+boto3==1.34.119
+botocore==1.34.119
 click==8.1.7
+coverage==5.3.1
 data-gradients==0.3.1
 datasets==2.14.5
 diffusers[torch]==0.21.4
 easydict==1.10
 einops==0.3.2
-ffmpeg==1.4
 ftfy==6.1.1
 hydra-core==1.3.0
 imageio[ffmpeg]==2.31.5
 imagesize==1.4.1
+jinja2==3.0.3
+keyrings.envvars==1.1.0; python_version >= '3.9' # used only by CI
 kornia==0.5.0
 librosa==0.10.1
 mmcv==2.1.0
 mmdet==3.2.0
 mmpose==1.2.0
+mypy==0.991
 object-detection-metrics==0.4.post1
 openai-whisper==20230314
+pre-commit==3.5.0
 pycocotools==2.0.7
+pytest-cov==4.1.0
+pytest-xdist==3.3.1
 pytorch-lightning==1.6.0
 rapidfuzz==3.8.1
-regex==2023.12.25
+regex==2023.10.3
+ruamel-yaml==0.18.6
+schema==0.7.5
 scikit-image==0.21.0
 scikit-learn==1.1.3
 scipy==1.8.1
@@ -39,14 +50,18 @@ sentencepiece==0.2.0
 shapely==2.0.3
 soundfile==0.12.1
 stringcase==1.2.0
+tensorboard==2.13.0
 tflite==2.10.0
 thop==0.1.1.post2209072238
-timm==0.9.11
-tensorboard==2.13.0
+timm==1.0.3
 torchaudio==0.13.1
+torchmetrics==1.4.0.post0
 transformers==4.41.1
 treelib==1.6.1
-tucker-conv==1.0.1
+types-PyYAML==6.0.12.12
+types-pillow==10.2.0.20240213
+types-requests==2.31.0.6
+types-tabulate==0.9.0.20240106
 ultralytics==8.0.193
 webdataset==0.2.86
 yacs==0.1.8

qai_hub_models/labels/cityscapes_labels.txt

@@ -0,0 +1,19 @@
+road
+sidewalk
+building
+wall
+fence
+pole
+traffic light
+traffic sign
+vegetation
+terrain
+sky
+person
+rider
+car
+truck
+bus
+train
+motorcycle
+bicycle

qai_hub_models/labels/coco_labels.txt

@@ -0,0 +1,80 @@
+person
+bicycle
+car
+motorcycle
+airplane
+bus
+train
+truck
+boat
+traffic light
+fire hydrant
+stop sign
+parking meter
+bench
+bird
+cat
+dog
+horse
+sheep
+cow
+elephant
+bear
+zebra
+giraffe
+backpack
+umbrella
+handbag
+tie
+suitcase
+frisbee
+skis
+snowboard
+sports ball
+kite
+baseball bat
+baseball glove
+skateboard
+surfboard
+tennis racket
+bottle
+wine glass
+cup
+fork
+knife
+spoon
+bowl
+banana
+apple
+sandwich
+orange
+broccoli
+carrot
+hot dog
+pizza
+donut
+cake
+chair
+couch
+potted plant
+bed
+dining table
+toilet
+tv
+laptop
+mouse
+remote
+keyboard
+cell phone
+microwave
+oven
+toaster
+sink
+refrigerator
+book
+clock
+vase
+scissors
+teddy bear
+hair drier
+toothbrush