%0 Conference Paper %1 gienapp-etal-2020-efficient %A Gienapp, Lukas %A Stein, Benno %A Hagen, Matthias %A Potthast, Martin %B Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics %C Online %D 2020 %E Jurafsky, Dan %E Chai, Joyce %E Schluter, Natalie %E Tetreault, Joel %I Association for Computational Linguistics %K imported %P 5772--5781 %R 10.18653/v1/2020.acl-main.511 %T Efficient Pairwise Annotation of Argument Quality %U https://aclanthology.org/2020.acl-main.511 %X We present an efficient annotation framework for argument quality, a feature difficult to be measured reliably as per previous work. A stochastic transitivity model is combined with an effective sampling strategy to infer high-quality labels with low effort from crowdsourced pairwise judgments. The model's capabilities are showcased by compiling Webis-ArgQuality-20, an argument quality corpus that comprises scores for rhetorical, logical, dialectical, and overall quality inferred from a total of 41,859 pairwise judgments among 1,271 arguments. With up to 93\% cost savings, our approach significantly outperforms existing annotation procedures. Furthermore, novel insight into argument quality is provided through statistical analysis, and a new aggregation method to infer overall quality from individual quality dimensions is proposed.

@inproceedings{gienapp-etal-2020-efficient, abstract = {We present an efficient annotation framework for argument quality, a feature difficult to be measured reliably as per previous work. A stochastic transitivity model is combined with an effective sampling strategy to infer high-quality labels with low effort from crowdsourced pairwise judgments. The model{'}s capabilities are showcased by compiling Webis-ArgQuality-20, an argument quality corpus that comprises scores for rhetorical, logical, dialectical, and overall quality inferred from a total of 41,859 pairwise judgments among 1,271 arguments. With up to 93{\%} cost savings, our approach significantly outperforms existing annotation procedures. Furthermore, novel insight into argument quality is provided through statistical analysis, and a new aggregation method to infer overall quality from individual quality dimensions is proposed.}, added-at = {2024-10-02T10:38:17.000+0200}, address = {Online}, author = {Gienapp, Lukas and Stein, Benno and Hagen, Matthias and Potthast, Martin}, biburl = {https://puma.scadsai.uni-leipzig.de/bibtex/2aca0ebfcd6500087f2df08b9ac41c71f/scadsfct}, booktitle = {Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics}, doi = {10.18653/v1/2020.acl-main.511}, editor = {Jurafsky, Dan and Chai, Joyce and Schluter, Natalie and Tetreault, Joel}, interhash = {5b02718693074a3e4ce7610cb5ee61a4}, intrahash = {aca0ebfcd6500087f2df08b9ac41c71f}, keywords = {imported}, month = jul, pages = {5772--5781}, publisher = {Association for Computational Linguistics}, timestamp = {2024-10-02T10:38:17.000+0200}, title = {Efficient Pairwise Annotation of Argument Quality}, url = {https://aclanthology.org/2020.acl-main.511}, year = 2020 }