Face Detection Library Replacement (analogdevicesinc#287)

.gitignore

@@ -10,6 +10,7 @@
 /logs/
 /ninja-python-distributions
 /pip-selfcheck.json
+/pretrained/
 /sensitivity.csv
 /sensitivity.png
 /tensorflow/

datasets/vggface2.py

@@ -1,6 +1,6 @@
 ###################################################################################################
 #
-# Copyright (C) 2019-2023 Maxim Integrated Products, Inc. All Rights Reserved.
+# Copyright (C) 2019-2024 Maxim Integrated Products, Inc. All Rights Reserved.
 #
 # Maxim Integrated Products, Inc. Default Copyright Notice:
 # https://www.maximintegrated.com/en/aboutus/legal/copyrights.html
@@ -23,9 +23,8 @@
 from torchvision import transforms
 
 import cv2
+import face_detection
 import kornia.geometry.transform as GT
-import onnxruntime
-from hawk_eyes.face import RetinaFace
 from PIL import Image
 from skimage import transform as trans
 from tqdm import tqdm
@@ -100,19 +99,26 @@ def __extract_gt(self):
         """
         Extracts the ground truth from the dataset
         """
-        onnxruntime.set_default_logger_severity(3)  # suppress onnxruntime warnings
-        retina = RetinaFace(model_name='retina_l', conf=0.5)
+        detector = face_detection.build_detector("RetinaNetResNet50", confidence_threshold=.5,
+                                                 nms_iou_threshold=.4)
         img_paths = list(glob.glob(os.path.join(self.d_path + '/**/', '*.jpg'), recursive=True))
         nf_number = 0
-        n_words = 0
+        words_count = 0
         pickle_dict = {key: [] for key in ["boxes", "landmarks", "img_list", "lbl_list"]}
         pickle_dict["word2index"] = {}
 
         for jpg in tqdm(img_paths):
             boxes = []
             image = cv2.imread(jpg)
-            bboxes, lndmrks = retina.detect(image)
-            if len(bboxes) == 0:
+
+            img_max = max(image.shape[0], image.shape[1])
+            if img_max > 1320:
+                continue
+            bboxes, lndmrks = detector.batched_detect_with_landmarks(np.expand_dims(image, 0))
+            bboxes = bboxes[0]
+            lndmrks = lndmrks[0]
+
+            if (bboxes.shape[0] == 0) or (lndmrks.shape[0] == 0):
                 nf_number += 1
                 continue
 
@@ -126,8 +132,8 @@ def __extract_gt(self):
             lbl = os.path.relpath(dir_name, self.d_path)
 
             if lbl not in pickle_dict["word2index"]:
-                pickle_dict["word2index"][lbl] = n_words
-                n_words += 1
+                pickle_dict["word2index"][lbl] = words_count
+                words_count += 1
 
             pickle_dict["lbl_list"].append(lbl)
             pickle_dict["boxes"].append(boxes)

docs/FacialRecognitionSystem.md

@@ -38,7 +38,7 @@ To train the facedetection model, "scripts/train_facedet_tinierssd.sh" script ca
 
 To train a FaceID model for MAX7800x microcontrollers, there are multiple steps. As the MAX7800x FaceID models will be trained in a knowledge distillation fashion, the first step will be downloading a backbone checkpoint for the teacher model.
 
-The suggested teacher model is IR-152, but the other teacher models defined in "model_irse_drl.py" may be used as well. Please review the terms and conditions at face.evoLVe[3] repository, and download the checkpoint according to your teacher model selection.
+The suggested teacher model is IR-152, but the other teacher models defined in "model_irse_drl.py" may be used as well. Please review the terms and conditions at face.evoLVe[3] repository, and download the checkpoint according to your teacher model selection. Then, the checkpoint should be placed in a folder (e.g. "pretrained") in the root directory of the repository. By using the "--backbone-checkpoint" argument, the path to the checkpoint should be given to the training script.
 
 There are two FaceID models, one for the MAX78000 and one for the MAX78002. The MAX78000 one is named faceid_112, and it is a relatively lightweight model. To enable more performance on MAX78002, a more complex model was developed, which is named mobilefacenet_112. To train the FaceID models, "scripts/train_faceid_112.sh" and "scripts/train_mobilefacenet_112.sh" scripts can be used, respectivey. Training scripts will realize Dimensionality Reduction and Relation Based-Knowledge Knowledge Distillation steps automatically. A summary of Dimensionality Reduction and Relation-Based Knowledge Distillation can be found in the following sub-sections.
 

models/model_irse_drl.py

@@ -26,7 +26,9 @@
 ###################################################################################################
 """
 FaceID Teacher Model to be used for Knowledge Distillation
+See https://github.com/MaximIntegratedAI/ai8x-training/blob/develop/docs/FacialRecognitionSystem.md
 """
+import sys
 from collections import namedtuple
 
 import torch
@@ -292,14 +294,24 @@ def _initialize_weights(self):
                     m.bias.data.zero_()
 
 
-def ir_50(input_size=(112, 112),  # pylint: disable=unused-argument
-          dimensionality=64,
-          backbone_checkpoint=None, **kwargs):
-    """Constructs a ir-50 model.
+def create_model(input_size=(112, 112),  # pylint: disable=unused-argument
+                 dimensionality=64,
+                 backbone_checkpoint=None,
+                 model_name="ir", model_size=152, **kwargs):
     """
-    model = Backbone(input_size, 50, 'ir')
+    Model + DRL constructor
+    """
+    model = Backbone(input_size, model_size, model_name)
     if backbone_checkpoint is not None:
-        model.load_state_dict(torch.load(backbone_checkpoint, map_location=torch.device('cpu')))
+        try:
+            model.load_state_dict(torch.load(backbone_checkpoint,
+                                             map_location=torch.device('cpu')))
+        except FileNotFoundError:
+            print('Backbone checkpoint was not found. Please follow the '
+                  'instructions in the docs/FacialRecognitionSystem.md file, '
+                  'FaceID section to download the backbone checkpoint.',
+                  file=sys.stderr)
+            sys.exit()
     for param in model.parameters():
         param.requires_grad = False
     drl = DRL(dimensionality)
@@ -308,85 +320,70 @@ def ir_50(input_size=(112, 112),  # pylint: disable=unused-argument
     return ensemble
 
 
+def ir_50(input_size=(112, 112),  # pylint: disable=unused-argument
+          dimensionality=64,
+          backbone_checkpoint=None, **kwargs):
+    """
+    Constructs a ir-50 model.
+    """
+    model = create_model(input_size, dimensionality, backbone_checkpoint, "ir", 50)
+
+    return model
+
+
 def ir_101(input_size=(112, 112),  # pylint: disable=unused-argument
            dimensionality=64,
            backbone_checkpoint=None, **kwargs):
-    """Constructs a ir-101 model.
     """
-    model = Backbone(input_size, 100, 'ir')
-    if backbone_checkpoint is not None:
-        model.load_state_dict(torch.load(backbone_checkpoint, map_location=torch.device('cpu')))
-    for param in model.parameters():
-        param.requires_grad = False
-    drl = DRL(dimensionality)
-    ensemble = Ensemble(model, drl)
+    Constructs a ir-101 model.
+    """
+    model = create_model(input_size, dimensionality, backbone_checkpoint, "ir", 100)
 
-    return ensemble
+    return model
 
 
 def ir_152(input_size=(112, 112),  # pylint: disable=unused-argument
            dimensionality=64,
            backbone_checkpoint=None, **kwargs):
-    """Constructs a ir-152 model.
     """
-    model = Backbone(input_size, 152, 'ir')
-    if backbone_checkpoint is not None:
-        model.load_state_dict(torch.load(backbone_checkpoint, map_location=torch.device('cpu')))
-    for param in model.parameters():
-        param.requires_grad = False
-    drl = DRL(dimensionality)
-
-    ensemble = Ensemble(model, drl)
+    Constructs a ir-152 model.
+    """
+    model = create_model(input_size, dimensionality, backbone_checkpoint, "ir", 152)
 
-    return ensemble
+    return model
 
 
 def ir_se_50(input_size=(112, 112),  # pylint: disable=unused-argument
              dimensionality=64,
              backbone_checkpoint=None, **kwargs):
-    """Constructs a ir_se-50 model.
     """
-    model = Backbone(input_size, 50, 'ir_se')
-    if backbone_checkpoint is not None:
-        model.load_state_dict(torch.load(backbone_checkpoint, map_location=torch.device('cpu')))
-    for param in model.parameters():
-        param.requires_grad = False
-    drl = DRL(dimensionality)
-    ensemble = Ensemble(model, drl)
+    Constructs a ir_se-50 model.
+    """
+    model = create_model(input_size, dimensionality, backbone_checkpoint, "ir_se", 50)
 
-    return ensemble
+    return model
 
 
 def ir_se_101(input_size=(112, 112),  # pylint: disable=unused-argument
               dimensionality=64,
               backbone_checkpoint=None, **kwargs):
-    """Constructs a ir_se-101 model.
     """
-    model = Backbone(input_size, 100, 'ir_se')
-    if backbone_checkpoint is not None:
-        model.load_state_dict(torch.load(backbone_checkpoint, map_location=torch.device('cpu')))
-    for param in model.parameters():
-        param.requires_grad = False
-    drl = DRL(dimensionality)
-    ensemble = Ensemble(model, drl)
+    Constructs a ir_se-101 model.
+    """
+    model = create_model(input_size, dimensionality, backbone_checkpoint, "ir_se", 100)
 
-    return ensemble
+    return model
 
 
 def ir_se_152(input_size=(112, 112),  # pylint: disable=unused-argument
               dimensionality=64,
               backbone_checkpoint=None, **kwargs):
-    """Constructs a ir_se-152 model.
     """
-    model = Backbone(input_size, 152, 'ir_se')
-    if backbone_checkpoint is not None:
-        model.load_state_dict(torch.load(backbone_checkpoint, map_location=torch.device('cpu')))
-    for param in model.parameters():
-        param.requires_grad = False
-    drl = DRL(dimensionality)
-    ensemble = Ensemble(model, drl)
+    Constructs a ir_se-152 model.
+    """
+    model = create_model(input_size, dimensionality, backbone_checkpoint, "ir_se", 152)
 
-    return ensemble
+    return model
 
 
 models = [

requirements.txt

@@ -6,15 +6,13 @@ Pillow>=7
 PyYAML>=5.1.1
 albumentations>=1.3.0
 faiss-cpu==1.7.4
+face-detection==0.2.2
 h5py>=3.7.0
-hawk-eyes==2.1.0
 imutils==0.5.4
 kornia==0.6.8
 librosa>=0.7.2
 numba<0.50.0
 numpy>=1.22,<1.23
-onnx==1.15.0
-onnxruntime==1.7.0
 opencv-python>=4.4.0
 protobuf>=3.20.1,<4.0
 pycocotools==2.0.7