%0 Conference Paper %1 bevendorff-etal-2019-bias %A Bevendorff, Janek %A Hagen, Matthias %A Stein, Benno %A Potthast, Martin %B Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics %C Florence, Italy %D 2019 %E Korhonen, Anna %E Traum, David %E Màrquez, Llu\'ıs %I Association for Computational Linguistics %K imported %P 6301--6306 %R 10.18653/v1/P19-1634 %T Bias Analysis and Mitigation in the Evaluation of Authorship Verification %U https://aclanthology.org/P19-1634 %X The PAN series of shared tasks is well known for its continuous and high quality research in the field of digital text forensics. Among others, PAN contributions include original corpora, tailored benchmarks, and standardized experimentation platforms. In this paper we review, theoretically and practically, the authorship verification task and conclude that the underlying experiment design cannot guarantee pushing forward the state of the art---in fact, it allows for top benchmarking with a surprisingly straightforward approach. In this regard, we present a ``Basic and Fairly Flawed'' (BAFF) authorship verifier that is on a par with the best approaches submitted so far, and that illustrates sources of bias that should be eliminated. We pinpoint these sources in the evaluation chain and present a refined authorship corpus as effective countermeasure.

@inproceedings{bevendorff-etal-2019-bias, abstract = {The PAN series of shared tasks is well known for its continuous and high quality research in the field of digital text forensics. Among others, PAN contributions include original corpora, tailored benchmarks, and standardized experimentation platforms. In this paper we review, theoretically and practically, the authorship verification task and conclude that the underlying experiment design cannot guarantee pushing forward the state of the art{---}in fact, it allows for top benchmarking with a surprisingly straightforward approach. In this regard, we present a {``}Basic and Fairly Flawed{''} (BAFF) authorship verifier that is on a par with the best approaches submitted so far, and that illustrates sources of bias that should be eliminated. We pinpoint these sources in the evaluation chain and present a refined authorship corpus as effective countermeasure.}, added-at = {2024-10-02T10:38:17.000+0200}, address = {Florence, Italy}, author = {Bevendorff, Janek and Hagen, Matthias and Stein, Benno and Potthast, Martin}, biburl = {https://puma.scadsai.uni-leipzig.de/bibtex/20f54b486edc450ad0d38409e468848f9/scadsfct}, booktitle = {Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics}, doi = {10.18653/v1/P19-1634}, editor = {Korhonen, Anna and Traum, David and M{\`a}rquez, Llu{\'\i}s}, interhash = {38dd9ff57bb4b2f2f2b89ce0a38dc94f}, intrahash = {0f54b486edc450ad0d38409e468848f9}, keywords = {imported}, month = jul, pages = {6301--6306}, publisher = {Association for Computational Linguistics}, timestamp = {2024-10-02T10:38:17.000+0200}, title = {Bias Analysis and Mitigation in the Evaluation of Authorship Verification}, url = {https://aclanthology.org/P19-1634}, year = 2019 }