We present an efficient annotation framework for argument quality, a feature difficult to be measured reliably as per previous work. A stochastic transitivity model is combined with an effective sampling strategy to infer high-quality labels with low effort from crowdsourced pairwise judgments. The model's capabilities are showcased by compiling Webis-ArgQuality-20, an argument quality corpus that comprises scores for rhetorical, logical, dialectical, and overall quality inferred from a total of 41,859 pairwise judgments among 1,271 arguments. With up to 93\% cost savings, our approach significantly outperforms existing annotation procedures. Furthermore, novel insight into argument quality is provided through statistical analysis, and a new aggregation method to infer overall quality from individual quality dimensions is proposed.
%0 Conference Paper
%1 gienapp-etal-2020-efficient
%A Gienapp, Lukas
%A Stein, Benno
%A Hagen, Matthias
%A Potthast, Martin
%B Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics
%C Online
%D 2020
%E Jurafsky, Dan
%E Chai, Joyce
%E Schluter, Natalie
%E Tetreault, Joel
%I Association for Computational Linguistics
%K imported
%P 5772--5781
%R 10.18653/v1/2020.acl-main.511
%T Efficient Pairwise Annotation of Argument Quality
%U https://aclanthology.org/2020.acl-main.511
%X We present an efficient annotation framework for argument quality, a feature difficult to be measured reliably as per previous work. A stochastic transitivity model is combined with an effective sampling strategy to infer high-quality labels with low effort from crowdsourced pairwise judgments. The model's capabilities are showcased by compiling Webis-ArgQuality-20, an argument quality corpus that comprises scores for rhetorical, logical, dialectical, and overall quality inferred from a total of 41,859 pairwise judgments among 1,271 arguments. With up to 93\% cost savings, our approach significantly outperforms existing annotation procedures. Furthermore, novel insight into argument quality is provided through statistical analysis, and a new aggregation method to infer overall quality from individual quality dimensions is proposed.
@inproceedings{gienapp-etal-2020-efficient,
abstract = {We present an efficient annotation framework for argument quality, a feature difficult to be measured reliably as per previous work. A stochastic transitivity model is combined with an effective sampling strategy to infer high-quality labels with low effort from crowdsourced pairwise judgments. The model{'}s capabilities are showcased by compiling Webis-ArgQuality-20, an argument quality corpus that comprises scores for rhetorical, logical, dialectical, and overall quality inferred from a total of 41,859 pairwise judgments among 1,271 arguments. With up to 93{\%} cost savings, our approach significantly outperforms existing annotation procedures. Furthermore, novel insight into argument quality is provided through statistical analysis, and a new aggregation method to infer overall quality from individual quality dimensions is proposed.},
added-at = {2024-10-02T10:38:17.000+0200},
address = {Online},
author = {Gienapp, Lukas and Stein, Benno and Hagen, Matthias and Potthast, Martin},
biburl = {https://puma.scadsai.uni-leipzig.de/bibtex/2aca0ebfcd6500087f2df08b9ac41c71f/scadsfct},
booktitle = {Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics},
doi = {10.18653/v1/2020.acl-main.511},
editor = {Jurafsky, Dan and Chai, Joyce and Schluter, Natalie and Tetreault, Joel},
interhash = {5b02718693074a3e4ce7610cb5ee61a4},
intrahash = {aca0ebfcd6500087f2df08b9ac41c71f},
keywords = {imported},
month = jul,
pages = {5772--5781},
publisher = {Association for Computational Linguistics},
timestamp = {2024-10-02T10:38:17.000+0200},
title = {Efficient Pairwise Annotation of Argument Quality},
url = {https://aclanthology.org/2020.acl-main.511},
year = 2020
}
