%0 Journal Article %1 Schlatt2024-os %A Schlatt, Ferdinand %A Fröbe, Maik %A Scells, Harrisen %A Zhuang, Shengyao %A Koopman, Bevan %A Zuccon, Guido %A Stein, Benno %A Potthast, Martin %A Hagen, Matthias %D 2024 %I arXiv %K %T A systematic investigation of distilling large language models into cross-encoders for passage re-ranking %X Cross-encoders distilled from large language models (LLMs) are often more effective re-rankers than cross-encoders fine-tuned on manually labeled data. However, the distilled models usually do not reach their teacher LLM's effectiveness. To investigate whether best practices for fine-tuning cross-encoders on manually labeled data (e.g., hard-negative sampling, deep sampling, and listwise loss functions) can help to improve LLM ranker distillation, we construct and release a new distillation dataset: Rank-DistiLLM. In our experiments, cross-encoders trained on Rank-DistiLLM reach the effectiveness of LLMs while being orders of magnitude more efficient. Our code and data is available at https://github.com/webis-de/msmarco-llm-distillation.

@article{Schlatt2024-os, abstract = {Cross-encoders distilled from large language models (LLMs) are often more effective re-rankers than cross-encoders fine-tuned on manually labeled data. However, the distilled models usually do not reach their teacher LLM's effectiveness. To investigate whether best practices for fine-tuning cross-encoders on manually labeled data (e.g., hard-negative sampling, deep sampling, and listwise loss functions) can help to improve LLM ranker distillation, we construct and release a new distillation dataset: Rank-DistiLLM. In our experiments, cross-encoders trained on Rank-DistiLLM reach the effectiveness of LLMs while being orders of magnitude more efficient. Our code and data is available at https://github.com/webis-de/msmarco-llm-distillation.}, added-at = {2024-09-10T10:41:24.000+0200}, author = {Schlatt, Ferdinand and Fr{\"o}be, Maik and Scells, Harrisen and Zhuang, Shengyao and Koopman, Bevan and Zuccon, Guido and Stein, Benno and Potthast, Martin and Hagen, Matthias}, biburl = {https://puma.scadsai.uni-leipzig.de/bibtex/2de0c94908476d2bda358ee41352d6ce1/scadsfct}, interhash = {af69f0082a175351ecc57508cedf636b}, intrahash = {de0c94908476d2bda358ee41352d6ce1}, keywords = {}, publisher = {arXiv}, timestamp = {2024-09-10T10:47:32.000+0200}, title = {A systematic investigation of distilling large language models into cross-encoders for passage re-ranking}, year = 2024 }