%0 Conference Paper %1 10.1145/3340531.3412109 %A Gienapp, Lukas %A Stein, Benno %A Hagen, Matthias %A Potthast, Martin %B Proceedings of the 29th ACM International Conference on Information & Knowledge Management %C New York, NY, USA %D 2020 %I Association for Computing Machinery %K cumulative difficulty discounted evaluation, gain, information normalized retrieval, topic %P 2033–2036 %R 10.1145/3340531.3412109 %T Estimating Topic Difficulty Using Normalized Discounted Cumulated Gain %U https://doi.org/10.1145/3340531.3412109 %X Information retrieval evaluation has to consider the varying "difficulty" between topics. Topic difficulty is often defined in terms of the aggregated effectiveness of a set of retrieval systems to satisfy a respective information need. Current approaches to estimate topic difficulty come with drawbacks such as being incomparable across different experimental settings. We introduce a new approach to estimate topic difficulty, which is based on the ratio of systems that achieve an NDCG score that is better than a baseline formed as random ranking of the pool of judged documents. We modify the NDCG measure to explicitly reflect a system's divergence from this hypothetical random ranker. In this way we achieve relative comparability of topic difficulty scores across experimental settings as well as stability to outlier systems?features lacking in previous difficulty estimations. We reevaluate the TREC 2012 Web Track's ad hoc task to demonstrate the feasibility of our approach in practice. %@ 9781450368599

@inproceedings{10.1145/3340531.3412109, abstract = {Information retrieval evaluation has to consider the varying "difficulty" between topics. Topic difficulty is often defined in terms of the aggregated effectiveness of a set of retrieval systems to satisfy a respective information need. Current approaches to estimate topic difficulty come with drawbacks such as being incomparable across different experimental settings. We introduce a new approach to estimate topic difficulty, which is based on the ratio of systems that achieve an NDCG score that is better than a baseline formed as random ranking of the pool of judged documents. We modify the NDCG measure to explicitly reflect a system's divergence from this hypothetical random ranker. In this way we achieve relative comparability of topic difficulty scores across experimental settings as well as stability to outlier systems?features lacking in previous difficulty estimations. We reevaluate the TREC 2012 Web Track's ad hoc task to demonstrate the feasibility of our approach in practice.}, added-at = {2024-10-02T10:38:17.000+0200}, address = {New York, NY, USA}, author = {Gienapp, Lukas and Stein, Benno and Hagen, Matthias and Potthast, Martin}, biburl = {https://puma.scadsai.uni-leipzig.de/bibtex/24cd67611183ecfaaa427b32992eaef6a/scadsfct}, booktitle = {Proceedings of the 29th ACM International Conference on Information \& Knowledge Management}, doi = {10.1145/3340531.3412109}, interhash = {d44cc8ff9b443a69eed7b2b629d677cd}, intrahash = {4cd67611183ecfaaa427b32992eaef6a}, isbn = {9781450368599}, keywords = {cumulative difficulty discounted evaluation, gain, information normalized retrieval, topic}, location = {Virtual Event, Ireland}, numpages = {4}, pages = {2033–2036}, publisher = {Association for Computing Machinery}, series = {CIKM '20}, timestamp = {2024-10-02T10:38:17.000+0200}, title = {Estimating Topic Difficulty Using Normalized Discounted Cumulated Gain}, url = {https://doi.org/10.1145/3340531.3412109}, year = 2020 }