{
  "id": "2112.00585",
  "version": "v2",
  "published": "2021-12-01T15:55:04.000Z",
  "updated": "2022-03-30T10:58:03.000Z",
  "title": "Neural Emotion Director: Speech-preserving semantic control of facial expressions in \"in-the-wild\" videos",
  "authors": [
    "Foivos Paraperas Papantoniou",
    "Panagiotis P. Filntisis",
    "Petros Maragos",
    "Anastasios Roussos"
  ],
  "comment": "CVPR 2022 (oral). Project page: https://foivospar.github.io/NED/",
  "categories": [
    "cs.CV"
  ],
  "abstract": "In this paper, we introduce a novel deep learning method for photo-realistic manipulation of the emotional state of actors in \"in-the-wild\" videos. The proposed method is based on a parametric 3D face representation of the actor in the input scene that offers a reliable disentanglement of the facial identity from the head pose and facial expressions. It then uses a novel deep domain translation framework that alters the facial expressions in a consistent and plausible manner, taking into account their dynamics. Finally, the altered facial expressions are used to photo-realistically manipulate the facial region in the input scene based on an especially-designed neural face renderer. To the best of our knowledge, our method is the first to be capable of controlling the actor's facial expressions by even using as a sole input the semantic labels of the manipulated emotions, while at the same time preserving the speech-related lip movements. We conduct extensive qualitative and quantitative evaluations and comparisons, which demonstrate the effectiveness of our approach and the especially promising results that we obtain. Our method opens a plethora of new possibilities for useful applications of neural rendering technologies, ranging from movie post-production and video games to photo-realistic affective avatars.",
  "revisions": [
    {
      "version": "v2",
      "updated": "2022-03-30T10:58:03.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "facial expressions",
      "neural emotion director",
      "speech-preserving semantic control",
      "novel deep domain translation framework",
      "in-the-wild"
    ],
    "tags": [
      "github project"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}