%0 Conference Paper %1 Frobe2024-pf %A Fröbe, Maik %A Reimer, Jan Heinrich %A MacAvaney, Sean %A Deckers, Niklas %A Reich, Simon %A Bevendorff, Janek %A Stein, Benno %A Hagen, Matthias %A Potthast, Martin %B Proceedings of the Thirty-ThirdInternational Joint Conference on Artificial Intelligence %C California %D 2024 %I International Joint Conferences on Artificial Intelligence Organization %K topic_lifescience %T The information retrieval experiment platform (extended abstract) %X We have built TIREx, the information retrieval experiment platform, to promote standardized, reproducible, scalable, and blinded retrieval experiments. Standardization is achieved through integration with PyTerrier's interfaces and compatibility with ir\_datasets and ir\_measures. Reproducibility and scalability are based on the underlying TIRA framework, which runs dockerized software in a cloud-native execution environment. Using Docker images of 50 standard retrieval approaches, we evaluated all of them on 32 tasks (i.e., 1,600 runs) in less than a week on a midsize cluster (1,620 CPU cores and 24 GPUs), demonstrating multi-task scalability. Importantly, TIRA also enables blind evaluation of AI experiments, as the test data can be hidden from public access and the tested approaches run in a sandbox that prevents data leaks. Keeping the test data hidden from public access ensures that it cannot be used by third parties for LLM training, preventing future training-test leaks.

@inproceedings{Frobe2024-pf, abstract = {We have built TIREx, the information retrieval experiment platform, to promote standardized, reproducible, scalable, and blinded retrieval experiments. Standardization is achieved through integration with PyTerrier's interfaces and compatibility with ir\_datasets and ir\_measures. Reproducibility and scalability are based on the underlying TIRA framework, which runs dockerized software in a cloud-native execution environment. Using Docker images of 50 standard retrieval approaches, we evaluated all of them on 32 tasks (i.e., 1,600 runs) in less than a week on a midsize cluster (1,620 CPU cores and 24 GPUs), demonstrating multi-task scalability. Importantly, TIRA also enables blind evaluation of AI experiments, as the test data can be hidden from public access and the tested approaches run in a sandbox that prevents data leaks. Keeping the test data hidden from public access ensures that it cannot be used by third parties for LLM training, preventing future training-test leaks.}, added-at = {2024-09-10T10:41:24.000+0200}, address = {California}, author = {Fr{\"o}be, Maik and Reimer, Jan Heinrich and MacAvaney, Sean and Deckers, Niklas and Reich, Simon and Bevendorff, Janek and Stein, Benno and Hagen, Matthias and Potthast, Martin}, biburl = {https://puma.scadsai.uni-leipzig.de/bibtex/2d38ef9adbc8e1512ac7a527f282a434f/scadsfct}, booktitle = {Proceedings of the {Thirty-ThirdInternational} Joint Conference on Artificial Intelligence}, conference = {Thirty-Third International Joint Conference on Artificial Intelligence \{IJCAI-24\}}, interhash = {b18934e80bafc0d9a88bc53a3f9542d1}, intrahash = {d38ef9adbc8e1512ac7a527f282a434f}, keywords = {topic_lifescience}, location = {Jeju, South Korea}, month = aug, publisher = {International Joint Conferences on Artificial Intelligence Organization}, timestamp = {2024-09-10T12:00:15.000+0200}, title = {The information retrieval experiment platform (extended abstract)}, year = 2024 }