Work on new model

Worked on the new deep learning model, but now realizing that it won't really work well for our application, so I'm switching to the older way of doing it.

.idea/.gitignore

+# Default ignored files
+/workspace.xml
\ No newline at end of file

.idea/FLEX vision repo.iml

+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="TestRunnerService">
+    <option name="PROJECT_TEST_RUNNER" value="Unittests" />
+  </component>
+</module>
\ No newline at end of file

.idea/inspectionProfiles/profiles_settings.xml

+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>
\ No newline at end of file

.idea/misc.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7" project-jdk-type="Python SDK" />
+</project>
\ No newline at end of file

.idea/modules.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/FLEX vision repo.iml" filepath="$PROJECT_DIR$/.idea/FLEX vision repo.iml" />
+    </modules>
+  </component>
+</project>
\ No newline at end of file

.idea/vcs.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>
\ No newline at end of file

README.md

 # FLEX Vision
 
-The vision portion of the FLEX team project, responsible for detecting and authenticating users' faces.
\ No newline at end of file
+The vision portion of the FLEX team project, responsible for detecting and authenticating users' faces.
+
+# Installation
+1. Install OpenCV
+1. Run `pip install -U scikit-learn`
+
+# Installing OpenCV on the Raspberry PI
+I followed [these instructions](https://www.pyimagesearch.com/2018/09/26/install-opencv-4-on-your-raspberry-pi/) with modifications in the following places:
+ - In Step #4, replace `sudo pip install virtualenv virtualenvwrapper` with `sudo pip3 install virtualenv virtualenvwrapper` to install it on python 3.
\ No newline at end of file

authenticate_user.py

 """
 Methods for authenticating a user.
 """
-
+import os
 import time
 
 import cv2
@@ -23,6 +23,22 @@ def load_encodings(file_location: str):
     return load_database(file_location)
 
 
+def find_faces(grey, face_cascade: cv2.CascadeClassifier):
+    """
+    Finds the faces in the given image frame.
+    :param grey: The greyscale image.
+    :param face_cascade: The face cascade to be used for recognition.
+    :return: The face cascade classifier.
+    """
+    return face_cascade.detectMultiScale(
+        grey,
+        scaleFactor=1.1,
+        minNeighbors=5,
+        minSize=(30, 30),
+        flags=cv2.CASCADE_SCALE_IMAGE
+    )
+
+
 def determine_identity(face_encoding, known_faces):
     """
     "Determines the most likely identity of a single face. Returns the user id.
@@ -51,7 +67,7 @@ def check_recognized_users(recognized_user_counts):
     return recognized_users
 
 
-def draw_rectanges_and_user_ids(image_frame, conversion: float, box_user_id_map: dict):
+def draw_rectangles_and_user_ids(image_frame, conversion: float, box_user_id_map: dict):
     """Draws the rectangles and user_ids onto the video stream so anyone viewing the stream could see them."""
     if box_user_id_map and len(box_user_id_map) > 0:
         for ((top, right, bottom, left), user_id) in box_user_id_map.items():
@@ -78,38 +94,34 @@ def recognize_user(known_faces: dict, encoding_model: str = "hog", image_flip: i
     recognized_user = None
     video_stream = common.start_video_stream(0)
 
+    # TODO get this
+    face_cascade = common.load_cascade(os.path.join(common.CASCADE_DIR, "haarcascade_frontalface_default.xml"))
+
     # Determine the time at which we will time out. Equal to current time + timeout.
     timeout_time: float = time.time() + TIMEOUT
     while time.time() < timeout_time:
-        # Read a image_frame from the video stream.
-        ret, image_frame = video_stream.read()
-        if image_flip is not None:
-            image_frame = cv2.flip(image_frame, image_flip)
-
-        # Convert input from BGR to RGB
-        cv2.cvtColor(image_frame, cv2.COLOR_BGR2RGB)
-        # Resize image to width of 750 PX to speed up processing.
-        rgb_image = imutils.resize(image_frame, width=750)
-        r = image_frame.shape[1] / float(rgb_image.shape[1])
-
-        # Detect the location of each face and determine the boxes in which they lie
-        boxes = face_recognition.face_locations(
-            rgb_image, model=encoding_model)
+        # Step 1: Image processing before we even get started with facial recognition.
+        grey, image_frame, r = process_next_image(video_stream, image_flip)
+
+        # Step 2: Detect locations of images.
+        boxes = find_faces(grey, face_cascade)
+
         # Compute the facial embeddings (the encoding) at
         # each of the locations found in the previous line.
-        encodings = face_recognition.face_encodings(rgb_image, boxes)
-
-        box_user_id_mapping = {}
-        for (i, encoding) in enumerate(encodings):
-            user_id: str = determine_identity(encoding, known_faces)
-            if user_id:
-                if user_id not in recognized_users_count:
-                    recognized_users_count[user_id] = 0
-                recognized_users_count[user_id] += 1
-                box_user_id_mapping[boxes[i]] = user_id
+        # encodings = face_recognition.face_encodings(grey, boxes)
+        #
+        # box_user_id_mapping = {}
+        # for (i, encoding) in enumerate(encodings):
+        #     user_id: str = determine_identity(encoding, known_faces)
+        #     if user_id:
+        #         if user_id not in recognized_users_count:
+        #             recognized_users_count[user_id] = 0
+        #         recognized_users_count[user_id] += 1
+        #         box_user_id_mapping[boxes[i]] = user_id
 
         if draw_rectangles:
-            draw_rectanges_and_user_ids(image_frame, r, box_user_id_mapping)
+            # draw_rectangles_and_user_ids(image_frame, r, box_user_id_mapping)
+            draw_rectangles_and_user_ids(image_frame, r, {box: "unknown" for box in boxes})
 
         # Now check if we have already positively identified a user enough times
         recognized_users = check_recognized_users(recognized_users_count)
@@ -125,6 +137,25 @@ def recognize_user(known_faces: dict, encoding_model: str = "hog", image_flip: i
     return recognized_user
 
 
+def process_next_image(video_stream, image_flip: int = None):
+    """
+    Processes the next image on the given video stream.
+    :param video_stream: The video stream.
+    :param image_flip: The integer way in which to flip the image if it will need flipping.
+    :return: A tuple of three elements: the processed greyscale image, the original read image and the r ratio.
+    """
+    # Read a image_frame from the video stream.
+    ret, image_frame = video_stream.read()
+    if image_flip is not None:
+        image_frame = cv2.flip(image_frame, image_flip)
+    # Convert input from BGR to RGB
+    grey = cv2.cvtColor(image_frame, cv2.COLOR_BGR2RGB)
+    # Resize image to width of 750 PX to speed up processing.
+    grey = imutils.resize(grey, width=750)
+    r = image_frame.shape[1] / float(grey.shape[1])
+    return grey, image_frame, r
+
+
 # If this program is the main program, authenticate the user.
 if __name__ == "__main__":
     import argparse;

common.py

@@ -3,7 +3,9 @@ import os
 import cv2
 
 DATA_DIR = "./data"
-DATABASE_LOC = os.path.join(DATA_DIR, "faces.pickle")
+DATASET_DIR = "./dataset"
+EMBEDDINGS_LOC = os.path.join(DATA_DIR, "embeddings.pickle")
+RECOGNITION_DATABASE_LOC = os.path.join(DATA_DIR, "recognition.pickle")
 CASCADE_DIR = "./cascades"
 # Output location for pickle database files
 OUTPUT_DIR = "./output"

data_handler.py

@@ -5,6 +5,27 @@ General IO for pickle database operations.
 import pickle
 
 
+def get_user_ids_in_database(database: dict) -> list:
+    """
+    Gets all the user_ids in the given database.
+    :param database: The database to look through.
+    :return: All the user_ids in the database.
+    """
+    return list(database.keys())
+
+
+def get_encodings_in_database(database: dict):
+    """
+    Gets a list of all encodings in the given database.
+    :param database: The database dictionary.
+    :return: All the encodings
+    """
+    result = []
+    for encodings in database.values():
+        result.extend(encodings)
+    return result
+
+
 def load_database(file_location: str):
     """
     Attempts to load the pickle database at the given file location
@@ -20,7 +41,7 @@ def load_database(file_location: str):
     return file_content
 
 
-def write_database(output_file: str, database_content: dict) -> None:
+def write_database(output_file: str, database_content) -> None:
     """
     Writes the dictionary database to the given file location
     :param output_file: The location of the file to be outputted on.

register_user.py

@@ -26,7 +26,7 @@ def process_image(image, encoding_model: str = "hog"):
     return face_recognition.face_encodings(image_rgb, [boxes[0]]) if boxes and len(boxes) > 0 else []
 
 
-def register_user(user_id: str, dataset_dir: str, encoding_model="hog", database_loc: str = common.DATABASE_LOC,
+def register_user(user_id: str, dataset_dir: str, encoding_model="hog", database_loc: str = common.EMBEDDINGS_LOC,
                   show_output: bool = False):
     """
     Function for registering a new user using the given video source. If video source isn't provided, then the camera

register_userv2.py

@@ -35,7 +35,8 @@ def find_best_match(faces, image, min_confidence: float = 0.5):
     best_match = None
 
     if len(faces) > 0:
-        # Assume that each image has only one face, so take the bounding box with the largest probability of being a face.
+        # Assume that each image has only one face, so take the bounding box with the largest probability of being a
+        # face.
         i = numpy.argmax(faces[0, 0, :, 2])
         confidence = faces[0, 0, i, 2]
 
@@ -117,7 +118,7 @@ def process_dataset(directory_location: str, detector_dir: str = common.FACE_DET
             if facial_embeddings is not None and len(facial_embeddings) > 0:
                 if current_user_id not in result_database:
                     result_database[current_user_id] = []
-                result_database[current_user_id].append(facial_embeddings)
+                result_database[current_user_id].append(facial_embeddings.flatten())
 
     if file_output is not None:
         data_handler.write_database(file_output, result_database)
@@ -130,8 +131,10 @@ if __name__ == "__main__":
 
     parser = argparse.ArgumentParser(
         description="Registers users' facial encodings from a dataset of images containing their face.")
-    parser.add_argument("dataset", type=str, help="Location of the dataset which should be processed.")
-    parser.add_argument("output", type=str,
-                        help="Location of the output pickle database file to which the encodings should be written.")
+    parser.add_argument("--dataset", "-d", type=str, help="Location of the dataset which should be processed.",
+                        default=common.DATASET_DIR)
+    parser.add_argument("--output", "-o", type=str,
+                        help="Location of the output pickle database file to which the encodings should be written.",
+                        default=common.EMBEDDINGS_LOC)
     args = parser.parse_args()
     process_dataset(args.dataset, show_output=True, file_output=args.output)

train_model.py

+"""
+Responsible for training the machine learning model for recognizing faces.
+"""
+
+
+from sklearn.preprocessing import LabelEncoder
+from sklearn.svm import SVC
+
+import common
+import data_handler
+
+def train_and_save(facial_embeddings_database: str= common.EMBEDDINGS_LOC, output_file: str = common.RECOGNITION_DATABASE_LOC) -> None:
+    """
+    Trains the database using the given facial embeddings database and outputs the results to file.
+    :param facial_embeddings_database: The facial embedding database location.
+    :param output_file: The file location for the output of the database.
+    :return: None
+    """
+    database = data_handler.load_database(facial_embeddings_database)
+    data_handler.write_database(output_file, train_model(database))
+
+
+def train_model(facial_embeddings: dict) -> SVC:
+    """
+    Trains the model for the given database
+    :param facial_embeddings_database: The location of the pickle database.
+    :param output_file: File location where to output the pickle database of facial recognitions.
+    :return:
+    """
+    label_encoder = LabelEncoder()
+    user_id_repeat_list = []
+    for user_id, encodings in facial_embeddings.items():
+        user_id_repeat_list.extend([user_id for x in range(len(encodings))])
+
+    # The facial_embeddings
+    labels = label_encoder.fit_transform(user_id_repeat_list)
+
+    recognizer = SVC(C=1.0, kernel="linear", probability=True)
+    # TODO not too sure this line does what is intended.
+    recognizer.fit(data_handler.get_encodings_in_database(facial_embeddings), labels)
+
+    return recognizer
+
+if __name__ == "__main__":
+    train_and_save()
\ No newline at end of file