@inbook{2d50903266c146039c71384f41aecd5e,
title = "Non-deterministic Behavior of Ranking-Based Metrics When Evaluating Embeddings",
abstract = "Embedding data into vector spaces is a very popular strategy of pattern recognition methods. When distances between embeddings are quantized, performance metrics become ambiguous. In this paper, we present an analysis of the ambiguity quantized distances introduce and provide bounds on the effect. We demonstrate that it can have a measurable effect in empirical data in state-of-the-art systems. We also approach the phenomenon from a computer security perspective and demonstrate how someone being evaluated by a third party can exploit this ambiguity and greatly outperform a random predictor without even access to the input data. We also suggest a simple solution making the performance metrics, which rely on ranking, totally deterministic and impervious to such exploits.",
keywords = "Adversarial, Deterministic, mAP, Performance evaluation, Precision, Ranking, Recall, Word spotting",
author = "Anguelos Nicolaou and Sounak Dey and Vincent Christlein and Andreas Maier and Dimosthenis Karatzas",
note = "Funding Information: Acknowledgments. This work has been partially supported by the European fund for regional development, grant-nr. 211 and the Spanish project TIN2017-89779-P. The contents of this publication are the sole responsibility of the authors. Publisher Copyright: {\textcopyright} Springer Nature Switzerland AG 2019.",
year = "2019",
doi = "10.1007/978-3-030-23987-9\_5",
language = "English",
isbn = "9783030239862",
series = "Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)",
pages = "71--82",
editor = "Bertrand Kerautret and Miguel Colom and Daniel Lopresti and Pascal Monasse and Hugues Talbot",
booktitle = "Reproducible Research in Pattern Recognition - Second International Workshop, RRPR 2018, Revised Selected Papers",
}