Publications

@inproceedings{2019-Hickson-ECFEUEC,

title = {Eyemotion: Classifying Facial Expressions in VR Using Eye-Tracking Cameras},

author = {S. Hickson and N. Dufour and A. Sud and V. Kwatra and I. Essa},

url = {https://ieeexplore.ieee.org/document/8658392

https://ai.google/research/pubs/pub46291},

doi = {10.1109/WACV.2019.00178},

issn = {1550-5790},

year  = {2019},

date = {2019-01-01},

urldate = {2019-01-01},

booktitle = {IEEE Winter Conference on Applications of Computer Vision (WACV)},

pages = {1626-1635},

abstract = {One of the main challenges of social interaction in virtual reality settings is that head-mounted displays occlude a large portion of the face, blocking facial expressions and thereby restricting social engagement cues among users. We present an algorithm to automatically infer expressions by analyzing only a partially occluded face while the user is engaged in a virtual reality experience. Specifically, we show that images of the user's eyes captured from an IR gaze-tracking camera within a VR headset are sufficient to infer a subset of facial expressions without the use of any fixed external camera. Using these inferences, we can generate dynamic avatars in real-time which function as an expressive surrogate for the user. We propose a novel data collection pipeline as well as a novel approach for increasing CNN accuracy via personalization. Our results show a mean accuracy of 74% (F1 of 0.73) among 5 'emotive' expressions and a mean accuracy of 70% (F1 of 0.68) among 10 distinct facial action units, outperforming human raters.

},

keywords = {audio-video fusion, face & gesture, face processing, multimodal interfaces, WACV},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2003-Ruddarraju-PUIUVT,

title = {Perceptual User Interfaces using Vision-Based Eye Tracking},

author = {Ravi Ruddarraju and Antonio Haro and Kris Nagel and Quan Tran and Irfan Essa and Gregory Abowd and Elizabeth Mynatt},

url = {https://doi.org/10.1145/958432.958475},

doi = {10.1145/958432.958475},

year  = {2003},

date = {2003-01-01},

urldate = {2003-01-01},

booktitle = {International Conference on Multimodal Interfaces (ICMI)},

address = {Vancouver, Canada},

abstract = {We present a multi-camera vision-based eye tracking method to robustly locate and track user's eyes as they interact with an application. We propose enhancements to various vision-based eye-tracking approaches, which include (a) the use of multiple cameras to estimate head pose and increase coverage of the sensors and (b) the use of probabilistic measures incorporating Fisher's linear discriminant to robustly track the eyes under varying lighting conditions in real-time. We present experiments and quantitative results to demonstrate the robustness of our eye tracking in two application prototypes.



},

keywords = {computer vision, eye-tracking, ICMI, multimodal interfaces},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{1994-Pentland-VGIA,

title = {Visually guided interaction and animation},

author = {Alex Pentland and Stan Sclaroff and Trevor Darrell and Irfan Essa and Ali Azarbayejani and Thad Starner},

year  = {1994},

date = {1994-01-01},

urldate = {1994-01-01},

booktitle = {Asilomar Conference on Signals, Systems, and Computers},

volume = {2},

pages = {1287 - 1291},

address = {Pacific Grove, CA},

keywords = {computer animation, human-computer interaction, multimodal interfaces},

pubstate = {published},

tppubtype = {inproceedings}

}