{
  "id": "2107.09562",
  "version": "v1",
  "published": "2021-07-20T15:26:09.000Z",
  "updated": "2021-07-20T15:26:09.000Z",
  "title": "Characterizing Generalization under Out-Of-Distribution Shifts in Deep Metric Learning",
  "authors": [
    "Timo Milbich",
    "Karsten Roth",
    "Samarth Sinha",
    "Ludwig Schmidt",
    "Marzyeh Ghassemi",
    "Björn Ommer"
  ],
  "categories": [
    "cs.LG",
    "cs.CV"
  ],
  "abstract": "Deep Metric Learning (DML) aims to find representations suitable for zero-shot transfer to a priori unknown test distributions. However, common evaluation protocols only test a single, fixed data split in which train and test classes are assigned randomly. More realistic evaluations should consider a broad spectrum of distribution shifts with potentially varying degree and difficulty. In this work, we systematically construct train-test splits of increasing difficulty and present the ooDML benchmark to characterize generalization under out-of-distribution shifts in DML. ooDML is designed to probe the generalization performance on much more challenging, diverse train-to-test distribution shifts. Based on our new benchmark, we conduct a thorough empirical analysis of state-of-the-art DML methods. We find that while generalization tends to consistently degrade with difficulty, some methods are better at retaining performance as the distribution shift increases. Finally, we propose few-shot DML as an efficient way to consistently improve generalization in response to unknown test shifts presented in ooDML. Code available here: https://github.com/Confusezius/Characterizing_Generalization_in_DeepMetricLearning.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2021-07-20T15:26:09.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "deep metric learning",
      "out-of-distribution shifts",
      "characterizing generalization",
      "priori unknown test distributions",
      "diverse train-to-test distribution shifts"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}