Publications

@inproceedings{2021-Zhang-TNOIMTI,

title = {Text as Neural Operator: Image Manipulation by Text Instruction},

author = {Tianhao Zhang and Hung-Yu Tseng and Lu Jiang and Weilong Yang and Honglak Lee and Irfan Essa},

url = {https://dl.acm.org/doi/10.1145/3474085.3475343

https://arxiv.org/abs/2008.04556},

doi = {10.1145/3474085.3475343},

year  = {2021},

date = {2021-10-01},

urldate = {2021-10-01},

booktitle = {ACM International Conference on Multimedia (ACM-MM)},

publisher = {ACM Press},

abstract = {In recent years, text-guided image manipulation has gained increasing attention in the multimedia and computer vision community. The input to conditional image generation has evolved from image-only to multimodality. In this paper, we study a setting that allows users to edit an image with multiple objects using complex text instructions to add, remove, or change the objects. The inputs of the task are multimodal including (1) a reference image and (2) an instruction in natural language that describes desired modifications to the image. We propose a GAN-based method to tackle this problem. The key idea is to treat text as neural operators to locally modify the image feature. We show that the proposed model performs favorably against recent strong baselines on three public datasets. Specifically, it generates images of greater fidelity and semantic relevance, and when used as a image query, leads to better retrieval performance.},

keywords = {computer vision, generative media, google, multimedia},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{2006-Kim-IMGVN,

title = {Interactive Mosaic Generation for Video Navigation},

author = {Kihwan Kim and Irfan Essa and Gregory Abowd },

url = {https://doi.org/10.1145/1180639.1180776},

doi = {10.1145/1180639.1180776},

year  = {2006},

date = {2006-10-01},

urldate = {2006-10-01},

booktitle = {ACM International Conference on Multimedia (ACM-MM)},

address = {Santa Barbara,CA,USA},

abstract = {Navigation through large multimedia collections that include videos and images still remains cumbersome. In this paper, we introduce a novel method to visualize and navigate through the collection by creating a mosaic image that visually represents the compilation. This image is generated by a labeling-based layout algorithm using various sizes of sample tile images from the collection. Each tile represents both the photographs and video files representing scenes selected by matching algorithms. This generated mosaic image provides a new way for thematic video and visually summarizes the videos. Users can generate these mosaics with some predefined themes and layouts, or base it on the results of their queries. Our approach supports automatic generation of these layouts by using meta-information such as color, time-line and existence of faces or manually generated annotated information from existing systems (e.g., the Family Video Archive).},

keywords = {ACMMM, computational video, multimedia},

pubstate = {published},

tppubtype = {inproceedings}

}