Pythonappendix

README.md

@@ -1,4 +1,14 @@
-# OpenFace 2.0.6: an open source facial behavior analysis toolkit
+![ ](https://camo.githubusercontent.com/3b74084ed00cb47fe3d2604372ebf842369e40fe/68747470733a2f2f692e696d6775722e636f6d2f6e5870564a65702e676966)
+
+# Eye Contact Detection built on top of OpenFace 2.0.6
+
+We present a tool made for the [USC Interaction Lab](http://robotics.usc.edu/interaction/img/ILAB_icon_yellow_nodecap.png) under the guidance of Lauren Klien, PhD. This tool uses state of the art prediction models to predict eye contact in infants with a 97% success rate. Please check the [eye_contact_files](https://github.com/pashpashpash/Eye-Detection-With-OpenFace/tree/master/eye_contact_files) directory for installation/running instructions.
+
+Project Contributors: John Dwyer, Nik Pash, Chuanxiu Xiong, Alec Schule, Christian Glover
+
+Research conducted under the guidance of Lauren Klien, PhD candidate at University of Southern California.
+
+<img src="http://robotics.usc.edu/interaction/img/ILAB_icon_yellow_nodecap.png" height="280">
 
 [![Build Status](https://travis-ci.org/TadasBaltrusaitis/OpenFace.svg?branch=master)](https://travis-ci.org/TadasBaltrusaitis/OpenFace)
 [![Build status](https://ci.appveyor.com/api/projects/status/8msiklxfbhlnsmxp/branch/master?svg=true)](https://ci.appveyor.com/project/TadasBaltrusaitis/openface/branch/master)

eye_contact_files/FaceLandmarkVid.cpp

@@ -0,0 +1,229 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
+// all rights reserved.
+//
+// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
+//
+// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.
+// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
+//
+// License can be found in OpenFace-license.txt
+
+//     * Any publications arising from the use of this software, including but
+//       not limited to academic journal and conference publications, technical
+//       reports and manuals, must cite at least one of the following works:
+//
+//       OpenFace 2.0: Facial Behavior Analysis Toolkit
+//       Tadas Baltru�aitis, Amir Zadeh, Yao Chong Lim, and Louis-Philippe Morency
+//       in IEEE International Conference on Automatic Face and Gesture Recognition, 2018
+//
+//       Convolutional experts constrained local model for facial landmark detection.
+//       A. Zadeh, T. Baltru�aitis, and Louis-Philippe Morency,
+//       in Computer Vision and Pattern Recognition Workshops, 2017.
+//
+//       Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
+//       Erroll Wood, Tadas Baltru�aitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling
+//       in IEEE International. Conference on Computer Vision (ICCV),  2015
+//
+//       Cross-dataset learning and person-specific normalisation for automatic Action Unit detection
+//       Tadas Baltru�aitis, Marwa Mahmoud, and Peter Robinson
+//       in Facial Expression Recognition and Analysis Challenge,
+//       IEEE International Conference on Automatic Face and Gesture Recognition, 2015
+//
+///////////////////////////////////////////////////////////////////////////////
+// FaceTrackingVid.cpp : Defines the entry point for the console application for tracking faces in videos.
+
+// Libraries for landmark detection (includes CLNF and CLM modules)
+#include "LandmarkCoreIncludes.h"
+#include "GazeEstimation.h"
+
+#include <SequenceCapture.h>
+#include <Visualizer.h>
+#include <VisualizationUtils.h>
+
+#include <cstdlib>
+#include <fstream>
+#include <iostream>
+
+#include <sstream>
+#include <string>
+using namespace std;
+
+#define INFO_STREAM( stream ) \
+std::cout << stream << std::endl
+
+#define WARN_STREAM( stream ) \
+std::cout << "Warning: " << stream << std::endl
+
+#define ERROR_STREAM( stream ) \
+std::cout << "Error: " << stream << std::endl
+
+static void printErrorAndAbort(const std::string & error)
+{
+	std::cout << error << std::endl;
+	abort();
+}
+
+#define FATAL_STREAM( stream ) \
+printErrorAndAbort( std::string( "Fatal error: " ) + stream )
+
+using namespace std;
+
+vector<string> get_arguments(int argc, char **argv)
+{
+
+	vector<string> arguments;
+
+	for (int i = 0; i < argc; ++i)
+	{
+		arguments.push_back(string(argv[i]));
+	}
+	return arguments;
+}
+
+int main(int argc, char **argv)
+{
+
+	vector<string> arguments = get_arguments(argc, argv);
+
+	// no arguments: output usage
+	if (arguments.size() == 1)
+	{
+		cout << "For command line arguments see:" << endl;
+		cout << " https://github.com/TadasBaltrusaitis/OpenFace/wiki/Command-line-arguments";
+		return 0;
+	}
+
+	LandmarkDetector::FaceModelParameters det_parameters(arguments);
+
+	// The modules that are being used for tracking
+	LandmarkDetector::CLNF face_model(det_parameters.model_location);
+	if (!face_model.loaded_successfully)
+	{
+		cout << "ERROR: Could not load the landmark detector" << endl;
+		return 1;
+	}
+
+	if (!face_model.eye_model)
+	{
+		cout << "WARNING: no eye model found" << endl;
+	}
+
+	// Open a sequence
+	Utilities::SequenceCapture sequence_reader;
+
+	// A utility for visualizing the results (show just the tracks)
+	Utilities::Visualizer visualizer(true, false, false, false);
+
+	// Tracking FPS for visualization
+	Utilities::FpsTracker fps_tracker;
+	fps_tracker.AddFrame();
+
+	int sequence_number = 0;
+    // std::string serverStart = "python3 predictorServer.py";
+    // std::system(serverStart.c_str()); // execute the UNIX command "ls -l >test.txt"
+
+	while (true) // this is not a for loop as we might also be reading from a webcam
+	{
+
+		// The sequence reader chooses what to open based on command line arguments provided
+		if (!sequence_reader.Open(arguments))
+			break;
+
+		INFO_STREAM("Device or file opened");
+
+		cv::Mat rgb_image = sequence_reader.GetNextFrame();
+
+		INFO_STREAM("Starting tracking");
+		int frameNum = 0;
+		while (!rgb_image.empty()) // this is not a for loop as we might also be reading from a webcam
+		{
+			frameNum++;
+			// Reading the images
+			cv::Mat_<uchar> grayscale_image = sequence_reader.GetGrayFrame();
+
+
+			// The actual facial landmark detection / tracking
+			bool detection_success = LandmarkDetector::DetectLandmarksInVideo(rgb_image, face_model, det_parameters, grayscale_image);
+
+			// Gaze tracking, absolute gaze direction
+			cv::Point3f gazeDirection0(0, 0, -1);
+			cv::Point3f gazeDirection1(0, 0, -1);
+			cv::Vec2f gaze_angle(0, 0);
+
+
+			// If tracking succeeded and we have an eye model, estimate gaze
+			if (detection_success && face_model.eye_model)
+			{
+				GazeAnalysis::EstimateGaze(face_model, gazeDirection0, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, true);
+				GazeAnalysis::EstimateGaze(face_model, gazeDirection1, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, false);
+				gaze_angle = GazeAnalysis::GetGazeAngle(gazeDirection0, gazeDirection1);
+			}
+
+			// Work out the pose of the head from the tracked model
+			cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
+
+			// Keeping track of FPS
+			fps_tracker.AddFrame();
+
+			// Displaying the tracking visualizations
+			visualizer.SetImage(rgb_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy);
+
+			// visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty, face_model.GetVisibilities());
+			// visualizer.SetObservationPose(pose_estimate, face_model.detection_certainty);
+			// visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_model.detection_certainty);
+
+			visualizer.SetFps(fps_tracker.GetFPS());
+			// detect key presses (due to pecularities of OpenCV, you can get it when displaying images)
+			char character_press = visualizer.ShowObservation();
+
+			// restart the tracker
+			if (character_press == 'r')
+			{
+				face_model.Reset();
+			}
+			// quit the application
+			else if (character_press == 'q')
+			{
+				return(0);
+			}
+
+			// Grabbing the next frame in the sequence
+			rgb_image = sequence_reader.GetNextFrame();
+
+			std::string gaze_angle_x = std::to_string(gaze_angle[0]);
+			std::string gaze_angle_y = std::to_string(gaze_angle[1]);
+			std::string pose_estimate_Tx= std::to_string(pose_estimate[0]);
+			std::string pose_estimate_Ty= std::to_string(pose_estimate[1]);
+			std::string pose_estimate_Tz= std::to_string(pose_estimate[2]);
+			std::string pose_estimate_Rx= std::to_string(pose_estimate[3]);
+			std::string pose_estimate_Ry= std::to_string(pose_estimate[4]);
+			std::string pose_estimate_Rz= std::to_string(pose_estimate[5]);
+
+			std::string command = std::to_string(frameNum) + " " + gaze_angle_x + " " + gaze_angle_y + " " + pose_estimate_Tx + " " + pose_estimate_Ty + " " + pose_estimate_Tz + " " + pose_estimate_Rx + " " + pose_estimate_Ry + " " + pose_estimate_Rz;
+			command = "python3 predictorClient.py " + command;
+			// std::string command2 = "clear" ;
+
+			// if(frameNum % 3 == 0) { //predict every 3rd frame
+            // // cout << "Pose_Estimate Tx = " << pose_estimate[0] << endl;
+            // // cout << "Pose_Estimate Ty = " << pose_estimate[1] << endl;
+            // // cout << "Pose_Estimate Tz = " << pose_estimate[2] << endl;
+            // // cout << "Gaze_Angle_X= " << gaze_angle[0] << endl;
+            cout << "Frame: " << frameNum << endl;
+            std::system(command.c_str()); // execute the UNIX command "ls -l >test.txt"
+        // }
+			// INFO_STREAM("TEST");
+		}
+
+		// Reset the model, for the next video
+		face_model.Reset();
+		sequence_reader.Close();
+
+		sequence_number++;
+
+	}
+	std::string killServer = "python3 predictorClient.py";
+	std::system(killServer.c_str()); // execute the UNIX command "ls -l >test.txt"
+
+	return 0;
+}

eye_contact_files/README.md

@@ -0,0 +1,34 @@
+# Eye Contact Detection Source Code
+This directory contains all of the resources you need to recompile the source code from scratch (in the [bin_replacement](https://github.com/pashpashpash/Eye-Detection-With-OpenFace/tree/master/eye_contact_files/bin_replacement) folder) as well as easily run the software without needing to compile (in the [ready2run_mac_binary](https://github.com/pashpashpash/Eye-Detection-With-OpenFace/tree/master/eye_contact_files/ready2run_mac_binary) folder).
+## Easy run on Mac
+Navigate to the [ready2run_mac_binary](https://github.com/pashpashpash/Eye-Detection-With-OpenFace/tree/master/eye_contact_files/ready2run_mac_binary) folder to run the executable code on Mac. This won't work on linux.
+## Source Code Installation and Recompilation
+To have full control over the source code (and be able to recompile it) you'll need to run all of the [OpenFace installation instructions](https://github.com/TadasBaltrusaitis/OpenFace/wiki) in the root directory, and then replace the `FaceLandmarkVid.cpp` file for the one our team wrote, located in this folder. Once you've successfully replaced the OpenFace `FaceLandmarkVid.cpp` for ours, navigate back to the root directory and recompile using
+```cmake```
+and
+```make```
+Once everything compiles, you should have a bin folder. Then, go back to [eye_contact_files](https://github.com/pashpashpash/Eye-Detection-With-OpenFace/tree/master/eye_contact_files) and copy the  bin_replacement/[predictorClient.py](https://github.com/pashpashpash/Eye-Detection-With-OpenFace/blob/master/eye_contact_files/bin_replacement/predictorClient.py),  bin_replacement/[predictorServer.py](https://github.com/pashpashpash/Eye-Detection-With-OpenFace/blob/master/eye_contact_files/bin_replacement/predictorServer.py), and bin_replacement/[dt.joblib](https://github.com/pashpashpash/Eye-Detection-With-OpenFace/blob/master/eye_contact_files/bin_replacement/dt.joblib) files into your newly made /bin/ folder inside of the root directory. Now you should have the compiled EyeDetection executable in `/bin/FaceLandmarkVid`, the two python files it calls as well as the prediction model `dt.joblib` file all inside of /bin/.
+
+## Running Instructions
+Once in the `/bin/` folder,  you'll need to start the python server that loads the prediction model that will be used by `FaceLandmarkVid` before running the binary,.
+```
+python3 predictorServer.py
+```
+
+If you get any errors, run a pip install on the missing dependencies. Next, open up a second terminal window and navigate to this same directory.
+
+While the `predictorServer.py` is running in another terminal window, run eye contact detection on a pre-recorded video by running
+```
+./FaceLandmarkVid -f video.mp4
+```
+
+To run eye contact detection on real-time video input (such as webcam), run
+```
+./FaceLandmarkVid -device 0
+```
+## Other notes
+You can get some degree of customization without needing to recompile the C++.
+
+For example, if you'd like to check against annotations while running a pre-recorded video, open the `predictorClient.py` file in a text-editor of your choice, and un-comment the corresponding blob.
+
+Similarly, to write predictions to a file in real-time, open the `predictorClient.py` file and un-comment the corresponding blob.

eye_contact_files/bin_replacement/predictorClient.py

@@ -0,0 +1,58 @@
+import socket
+import sys
+import pickle
+import csv
+
+HOST = '127.0.0.1'  # The server's hostname or IP address
+PORT = 8102      # The port used by the server
+
+class bcolors:
+    HEADER = '\033[95m'
+    OKBLUE = '\033[94m'
+    OKGREEN = '\033[92m'
+    WARNING = '\033[93m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    UNDERLINE = '\033[4m'
+
+with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+    s.connect((HOST, PORT))
+
+    args = sys.argv
+    frame = int(sys.argv[1])
+    del sys.argv[1]
+
+    data = pickle.dumps(args)
+    s.send(data)
+    data = s.recv(1024)
+    prediction = pickle.loads(data)
+
+    # # Uncomment this blob if you want the annotations exported to an output.csv
+    ## Make sure you make an empty output.csv file in this folder before running. The script adds rows to existing csv's.
+    # with open('output.csv', 'a') as fd:
+    #     writer = csv.writer(fd, delimiter = ' ', quotechar='|', quoting=csv.QUOTE_MINIMAL)
+    #     writer.writerow(str(prediction))
+    ## end of blob
+
+    # # Comment out this blob if running with realtime video.
+    # # This is for checking against annotations while running faceLandMarkVid on a video file.
+    # with open('009-annotations.csv') as csvfile:
+    #     readCSV = list(csv.reader(csvfile, delimiter=','))
+    #     if(readCSV[frame][0] != ''):
+    #         if(int(readCSV[frame][0]) == 0 or int(readCSV[frame][0]) == 1):
+    #             if(int(prediction) == int(readCSV[frame][0])):
+    #                 print (bcolors.OKGREEN + "Matches annotation"  + bcolors.ENDC)
+    #             else:
+    #                 print (bcolors.FAIL + "Doesn't match annotation"  + bcolors.ENDC)
+    #     else:
+    #         print (bcolors.OKBLUE + "No annotations found for this frame" + bcolors.ENDC)
+    # # end of blob
+
+    if(prediction == 0):
+        print ("MODEL PREDICTION " + bcolors.WARNING +"NOT LOOKING" +  bcolors.ENDC)
+    else:
+        print ("MODEL PREDICTION " + bcolors.OKGREEN +"LOOKING" +  bcolors.ENDC)
+    print ("–––––––––––––––––––––––––––––––")
+    s.close()
+    exit()

eye_contact_files/bin_replacement/predictorServer.py

@@ -0,0 +1,41 @@
+from sklearn.externals import joblib
+import sys
+import numpy as np
+import socket
+import pickle
+import csv
+
+def  predict(args,model,conn):
+    args.pop(0) #removes name of script
+    argsArray = np.array(args) #convert to array for passing as features
+    argsArray.reshape(1,-1) #convert to correct shape
+    argsArray = argsArray.astype(np.float64) #convert to float64
+
+    # pred =[0, 0]
+    pred = model.predict([argsArray])  #make prediction from arguments
+    # print("predict reached", pred[0])
+
+    data2 = pickle.dumps(pred[0])
+    conn.send(data2)
+
+    return pred[0] #return prediction
+
+model = joblib.load('dt.joblib')#load sci kit model
+
+HOST = '127.0.0.1'  # Standard loopback interface address (localhost)
+PORT = 8102   # Port to listen on (non-privileged ports are > 1023)
+
+
+with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+    s.bind((HOST, PORT))
+    s.listen(100)
+    while True:
+        conn, addr = s.accept()
+        try:
+            data = conn.recv(1024)
+            args = pickle.loads(data)
+            predict(args,model,conn)
+            # s.close()
+        except:
+            # s.close()
+            print("")