Commit e16a3ae8 authored by Kyle Anderson's avatar Kyle Anderson

Merge branch 'interface' into 'master'

Add Interface for React App

See merge request !3
parents 40c01470 9a8ba48e
# Ignore database files
# Created by,pycharm
# Edit at,pycharm
### PyCharm ###
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
# Reference:
# User-specific stuff
# Generated files
# Sensitive or high-churn files
# Gradle
# Gradle and Maven with auto-import
# When using Gradle or Maven with auto-import, you should exclude module files,
# since they will be recreated, and may cause churn. Uncomment if using
# auto-import.
# .idea/modules.xml
# .idea/*.iml
# .idea/modules
# *.iml
# *.ipr
# CMake
# Mongo Explorer plugin
# File-based project format
# IntelliJ
# mpeltonen/sbt-idea plugin
# JIRA plugin
# Cursive Clojure plugin
# Crashlytics plugin (for Android Studio and IntelliJ)
# Editor-based Rest Client
# Android studio 3.1+ serialized cache file
### PyCharm Patch ###
# Comment Reason:
# *.iml
# modules.xml
# .idea/misc.xml
# *.ipr
# Sonarlint plugin
# SonarQube Plugin
# Markdown Navigator plugin
### Python ###
# Byte-compiled / optimized / DLL files
# C extensions
# Distribution / packaging
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
# Installer logs
# Unit test / coverage reports
# Translations
# Scrapy stuff:
# Sphinx documentation
# PyBuilder
# pyenv
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
# celery beat schedule file
# SageMath parsed files
# Spyder project settings
# Rope project settings
# Mr Developer
# mkdocs documentation
# mypy
# Pyre type checker
# End of,pycharm
\ No newline at end of file
# Default ignored files
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<component name="TestRunnerService">
<option name="PROJECT_TEST_RUNNER" value="Unittests" />
\ No newline at end of file
<component name="InspectionProjectProfileManager">
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7" project-jdk-type="Python SDK" />
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<module fileurl="file://$PROJECT_DIR$/.idea/FLEX vision repo.iml" filepath="$PROJECT_DIR$/.idea/FLEX vision repo.iml" />
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
\ No newline at end of file
# FLEX Vision
The vision portion of the FLEX team project, responsible for detecting and authenticating users' faces.
# Installation
1. Install OpenCV
1. Run `pip install -U scikit-learn`
# Installing OpenCV on the Raspberry PI
I followed [these instructions]( with modifications in the following places:
- In Step #4, replace `sudo pip install virtualenv virtualenvwrapper` with `sudo pip3 install virtualenv virtualenvwrapper` to install it on python 3.
\ No newline at end of file
......@@ -2,54 +2,42 @@
Methods for authenticating a user.
from import VideoStream
import face_recognition
import imutils
import pickle
import time
import cv2
import face_recognition
import imutils
import common
import data_handler
# How long to wait before timing out and saying failed authentication.
TIMEOUT: float = 30.0
# Minimum number of frames in which a user must be recognized in order to be authenticated.
image_writer = None
USER_IDS_KEY: str = "names" # TODO change
USER_IDS_KEY: str = "user_ids"
def load_encodings(file_location: str):
"""Loads the encodings for faces from the given file location."""
with open(file_location, "rb") as encodings_file:
encodings = pickle.loads(
return encodings
def start_video_stream(camera: int):
"""Starts the video stream and returns the created stream.
Also waits for the video stream to open before returning it."""
video_stream = VideoStream(src=camera).start()
return video_stream
return data_handler.load_database(file_location)
def determine_identity(face_encoding, known_faces):
"""Determines the most likely identity of a single face. Returns the user id."""
matches = face_recognition.compare_faces(
known_faces["encodings"], face_encoding)
matched_user = ''
matched_user_id_count = {}
# If there is at least one match to a face in the database, figure out which one it is.
if True in matches:
matched_users = [user_index for (
user_index, is_match) in enumerate(matches) if is_match]
for i in matched_users:
user_id: str = known_faces[USER_IDS_KEY][i]
matched_user_id_count[user_id] = matched_user_id_count.get(user_id, 0) + 1
matched_user: str = max(matched_user_id_count,
"Determines the most likely identity of a single face. Returns the user id.
:param face_encoding: The encoding which needs identification.
:param known_faces: The database of known faces to use for searching.
:return: The string user_id of the recognized user.
recognized_users = {}
for (user_id, user_encodings) in known_faces.items():
matches = face_recognition.compare_faces(user_encodings, face_encoding)
# Count the number of occurrences of true.
recognized_users[user_id] = matches.count(True)
matched_user: str = max(recognized_users,
return matched_user
......@@ -63,10 +51,10 @@ def check_recognized_users(recognized_user_counts):
return recognized_users
def draw_rectanges_and_user_ids(image_frame, conversion: float, boxes, user_ids: list):
def draw_rectangles_and_user_ids(image_frame, conversion: float, box_user_id_map: dict):
"""Draws the rectangles and user_ids onto the video stream so anyone viewing the stream could see them."""
if boxes and user_ids and len(user_ids) > 0:
for ((top, right, bottom, left), user_id) in zip(boxes, user_ids):
if box_user_id_map and len(box_user_id_map) > 0:
for ((top, right, bottom, left), user_id) in box_user_id_map.items():
top = round(top * conversion)
right = round(right * conversion)
bottom = round(bottom * conversion)
......@@ -77,30 +65,22 @@ def draw_rectanges_and_user_ids(image_frame, conversion: float, boxes, user_ids:
# Find the top so we can put the text there.
y = top - 15 if top - 15 > 15 else top + 15
cv2.putText(image_frame, user_id, (left, y), cv2.FONT_HERSHEY_PLAIN, 0.75, (0, 255, 0), 2)
def display_frame(frame):
"""Displays the frame to the user."""
cv2.imshow("Frame", frame)
def recognize_user(encodings_location: str = "./encodings.pickle", encoding_model: str = "hog", image_flip: int = None,
def recognize_user(known_faces: dict, encoding_model: str = "hog", image_flip: int = None,
"""Attempts to recognize a user.
Returns the ID of the user if identified, or None if no users are identified.
Dictionary of the form { "user_id": #frames_recognized } to keep
track of how many times each user was recognized."""
Returns the ID of the user if identified, or None if no users are identified."""
recognized_users_count = {}
recognized_user = None
video_stream = start_video_stream(0)
known_faces = load_encodings(encodings_location)
video_stream = common.start_video_stream(0)
# Determine the time at which we will time out. Equal to current time + timeout.
timeout_time: float = time.time() + TIMEOUT
while time.time() < timeout_time:
# Read a image_frame from the video stream.
image_frame =
ret, image_frame =
if image_flip is not None:
image_frame = cv2.flip(image_frame, image_flip)
......@@ -113,25 +93,27 @@ def recognize_user(encodings_location: str = "./encodings.pickle", encoding_mode
# Detect the location of each face and determine the boxes in which they lie
boxes = face_recognition.face_locations(
rgb_image, model=encoding_model)
# Computer the facial embeddings (the encoding) at
# Compute the facial embeddings (the encoding) at
# each of the locations found in the previous line.
encodings = face_recognition.face_encodings(rgb_image, boxes)
for encoding in encodings:
box_user_id_mapping = {}
for (i, encoding) in enumerate(encodings):
user_id: str = determine_identity(encoding, known_faces)
if user_id:
if user_id not in recognized_users_count:
recognized_users_count[user_id] = 0
recognized_users_count[user_id] += 1
box_user_id_mapping[boxes[i]] = user_id
if draw_rectangles:
draw_rectanges_and_user_ids(image_frame, r, boxes, list(known_faces[USER_IDS_KEY]))
draw_rectangles_and_user_ids(image_frame, r, box_user_id_mapping)
# Now check if we have already positively identified a user enough times
recognized_users = check_recognized_users(recognized_users_count)
if len(recognized_users) > 0:
cv2.waitKey(1) # Required or else video stream doesn't really render.
cv2.waitKey(20) # Required or else video stream doesn't really render.
if recognized_users_count:
recognized_user = max(recognized_users_count,
......@@ -141,23 +123,46 @@ def recognize_user(encodings_location: str = "./encodings.pickle", encoding_mode
return recognized_user
def recognize_user_from_database(database_loc: str = common.DATABASE_LOC, encoding_model: str = "hog",
image_flip: int = None, draw_rectangles: bool = False):
Recognizes a user
:param database_loc: The database containing the face encodings for users.
:param encoding_model: The encoding model to be used for recognition.
:param image_flip: The type of image flip to be applied to the image, if it will be upside-down or horizontally inverted.
:param draw_rectangles: True to draw the rectangles to the screen, false otherwise.
:return: The recognized user's id, or None if no user was recognized.
return recognize_user(data_handler.load_database(database_loc), encoding_model=encoding_model,
# If this program is the main program, authenticate the user.
if __name__ == "__main__":
import argparse;
import argparse
parser = argparse.ArgumentParser(description="Facial Identification Options")
parser.add_argument("--encodings", "-e", type=str, help="File location of facial encodings.", required=False,
parser.add_argument("--model", "-m", type=str, help="Type of encoding method, either \"hog\" or \"cnn\". HOG is "
"faster, CNN is more accurate.", required=False,
default="hog", choices=["cnn", "hog"])
default=None, choices=["cnn", "hog"])
parser.add_argument("--flip", "-f", type=int,
help="Whether or not to flip the image vertically or horizontally. 0 to flip horizontally, 1 to flip vertically.",
help="Whether or not to flip the image vertically or horizontally. 0 to flip horizontally, "
"1 to flip vertically.",
required=False, default=None, choices=[0, 1])
parser.add_argument("--show", "-s", action="store_true",
help="Include this argument to have the image shown to you.", default=False)
args = parser.parse_args()
user = recognize_user(encoding_model=args.model, encodings_location=args.encodings, image_flip=args.flip,
args_dict = {}
if args.encodings is not None:
args_dict["encodings_location"] = args.encodings
if args.model is not None:
args_dict["encoding_model"] = args.model
user = recognize_user_from_database(**args_dict, image_flip=args.flip,
if user:
print(f"Recognized user {user}.")
import os
import cv2
DATA_DIR = "data"
DATASET_DIR = "dataset"
DATABASE_LOC = os.path.join(DATA_DIR, "database.pickle")
# Directory for the face detection model.
FACE_DETECTION_MODEL_DIR = os.path.join(RES_DIRECTORY, "face_detection_model")
EMBEDDINGS_PROCESSOR_LOC = os.path.join(RES_DIRECTORY, "openface_nn4.small2.v1.t7")
def display_frame(frame):
"""Displays the frame to the user."""
cv2.imshow("Frame", frame)
def start_video_stream(camera: int):
"""Starts the video stream and returns the created stream.
Also waits for the video stream to open before returning it."""
video_stream = cv2.VideoCapture(0)
return video_stream
def load_cascade(cascade_loc: str) -> cv2.CascadeClassifier:
Opens the cascade classifier at the given path.
:param cascade_loc: The file location of the cascade.
:return:The CascadeClassifier class.
return cv2.CascadeClassifier(cascade_loc)
def load_detector(proto_path: str, model_path: str):
Loads the caffe detector with the given proto text file and the model file.
:param proto_path: The path location of the prototext file.
:param model_path: The path to the caffe model.
:return: The detector.
return cv2.dnn.readNetFromCaffe(proto_path, model_path)
def load_embedding_model(model_path: str):
Loads the torch embedding model at the given location.
:param model_path: The path to the model.
:return: The embedding model
return cv2.dnn.readNetFromTorch(model_path)
CAFFE_MODEL_NAME = "res10_300x300_ssd_iter_140000.caffemodel"
PROTOTXT_NAME = "deploy.prototxt"
def load_detector_from_dir(detector_dir: str):
prototxt: str = os.path.join(detector_dir, PROTOTXT_NAME)
caffe_model: str = os.path.join(detector_dir, CAFFE_MODEL_NAME)
return load_detector(prototxt, caffe_model)
Creates a facial recognition profile for a new user.
General IO for pickle database operations.
import pickle
def get_user_ids_in_database(database: dict) -> list:
Gets all the user_ids in the given database.
:param database: The database to look through.
:return: All the user_ids in the database.
return list(database.keys())
def get_encodings_in_database(database: dict):
Gets a list of all encodings in the given database.
:param database: The database dictionary.
:return: All the encodings
result = []
for encodings in database.values():
return result
def load_database(file_location: str):
Attempts to load the pickle database at the given file location
:param file_location: String location of file to be loaded.
:return: The loaded pickle database.
file_content = {}
with open(file_location, "rb") as database:
file_content = pickle.load(database)
except (FileNotFoundError, EOFError):
file_content = {}
return file_content
def write_database(output_file: str, database_content) -> None:
Writes the dictionary database to the given file location
:param output_file: The location of the file to be outputted on.
:param database_content: The database content to be written to the file.
:return: None
if output_file and database_content and database_content is not None:
with open(output_file, "wb") as output:
pickle.dump(database_content, output)
API Interface to the modules in this project. To be used by the UI application.
import os
import authenticate_user as authenticate
import common
import register_user as register
def authenticate_user(base_dir: str = os.getcwd()) -> str:
Interface for authenticating a user with the given base directory location.
:param base_dir: The base directory, from which all default paths are relative. Defaults to current working directory.
:return: The user_id of the recognized user, or empty string if none was recognized.
recognized = authenticate.recognize_user_from_database(database_loc=os.path.join(base_dir, common.DATA_DIR))
return recognized if recognized is not None else ""
def register_user(base_dir: str = os.getcwd()) -> None:
Registers new added users in the database.
:param base_dir: The base directory, from which all default paths are relative. Defaults to current working directory.
:return: None
register.register_users_and_save(directory_location=os.path.join(base_dir, common.DATASET_DIR),
delete_images_on_complete=True, overwrite_data=False)
Creates a facial recognition profile for a new user.
import os
import cv2
import face_recognition
from imutils import paths as impaths
import common
import data_handler
def process_image(image, encoding_model: str = "hog"):
Processes a single image, returning the encoded face object
:param image: The image containing the face to be processed.
:param encoding_model: The encoding model, either CNN or HOG
:return: The processed facial recognition profile encoding.
image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) # Convert between the image formats
# Detect the coordinates of the boxes corresponding to faces in the input image
boxes = face_recognition.face_locations(image_rgb, model=encoding_model)
# Actually make the encodings for the face.
# Only want the first recognized face
return face_recognition.face_encodings(image_rgb, [boxes[0]]) if boxes and len(boxes) > 0 else []
def delete_file(file_path: str) -> None:
Deletes the file at the given location.
:param file_path: The path to the file.
:return: None
def register_user(user_id: str, dataset_dir: str, encoding_model="hog",
show_output: bool = False, delete_on_processed: bool = False):
Function for registering a new user using the given video source. If video source isn't provided, then the camera
on id 0 is used.
:param user_id: The user id for the user that is being registered.
:param dataset_dir: The directory location of pictures for the user.
:param encoding_model: The type of encoding model. Must be either "hog" or "cnn". HOG is faster, CNN is more thorough.
:param show_output: True to print console output for progress, false otherwise.
:param delete_on_processed: True to delete the image file after processing it, false otherwise.
:return: Encoded face that was detected, or None if no face was detected or if there was another error.
processed_images = []
for (i, filename) in enumerate(impaths.list_images(dataset_dir)):
# Might want to check file validity here at some point, but won't for now.
image = cv2.imread(filename)
if image is not None:
if show_output:
print(f"Processing image {i + 1} for user {user_id}")
processed = process_image(image, encoding_model=encoding_model)
if processed:
# Delete after we're done if we're supposed to.
if delete_on_processed:
return {user_id: processed_images} if len(processed_images) > 0 else None
def register_users_in_dir(directory_location: str, encoding_model: str = "hog", delete_images_on_complete: bool = False,
show_output: bool = False):