BACKGROUND: Image-based cancer classifiers suffer from a variety of problems which negatively affect their performance. For example, variation in image brightness or different cameras can already suffice to diminish performance. Ensemble solutions, where multiple model predictions are combined into one, can improve these problems. However, ensembles are computationally intensive and less transparent to practitioners than single model solutions. Constructing model soups, by averaging the weights of multiple models into a single model, could circumvent these limitations while still improving performance. OBJECTIVE: To investigate the performance of model soups for a dermoscopic melanoma-nevus skin cancer classification task with respect to (1) generalisation to images from other clinics, (2) robustness against small image changes and (3) calibration such that the confidences correspond closely to the actual predictive uncertainties. METHODS: We construct model soups by fine-tuning pre-trained models on seven different image resolutions and subsequently averaging their weights. Performance is evaluated on a multi-source dataset including holdout and external components. RESULTS: We find that model soups improve generalisation and calibration on the external component while maintaining performance on the holdout component. For robustness, we observe performance improvements for pertubated test images, while the performance on corrupted test images remains on par. CONCLUSIONS: Overall, souping for skin cancer classifiers has a positive effect on generalisation, robustness and calibration. It is easy for practitioners to implement and by combining multiple models into a single model, complexity is reduced. This could be an important factor in achieving clinical applicability, as less complexity generally means more transparency.
%0 Journal Article
%1 Maron2022-ld
%A Maron, Roman C
%A Hekler, Achim
%A Haggenmüller, Sarah
%A von Kalle, Christof
%A Utikal, Jochen S
%A Müller, Verena
%A Gaiser, Maria
%A Meier, Friedegund
%A Hobelsberger, Sarah
%A Gellrich, Frank F
%A Sergon, Mildred
%A Hauschild, Axel
%A French, Lars E
%A Heinzerling, Lucie
%A Schlager, Justin G
%A Ghoreschi, Kamran
%A Schlaak, Max
%A Hilke, Franz J
%A Poch, Gabriela
%A Korsing, Sören
%A Berking, Carola
%A Heppt, Markus V
%A Erdmann, Michael
%A Haferkamp, Sebastian
%A Schadendorf, Dirk
%A Sondermann, Wiebke
%A Goebeler, Matthias
%A Schilling, Bastian
%A Kather, Jakob N
%A Fröhling, Stefan
%A Lipka, Daniel B
%A Krieghoff-Henning, Eva
%A Brinker, Titus J
%D 2022
%I Elsevier BV
%J Eur. J. Cancer
%K Artificial Calibration; Deep Dermatology; Ensembles; Generalisation; Melanoma; Model Nevus; Robustness intelligence; learning; soups; topic_lifescience
%P 307--316
%T Model soups improve performance of dermoscopic skin cancer classifiers
%V 173
%X BACKGROUND: Image-based cancer classifiers suffer from a variety of problems which negatively affect their performance. For example, variation in image brightness or different cameras can already suffice to diminish performance. Ensemble solutions, where multiple model predictions are combined into one, can improve these problems. However, ensembles are computationally intensive and less transparent to practitioners than single model solutions. Constructing model soups, by averaging the weights of multiple models into a single model, could circumvent these limitations while still improving performance. OBJECTIVE: To investigate the performance of model soups for a dermoscopic melanoma-nevus skin cancer classification task with respect to (1) generalisation to images from other clinics, (2) robustness against small image changes and (3) calibration such that the confidences correspond closely to the actual predictive uncertainties. METHODS: We construct model soups by fine-tuning pre-trained models on seven different image resolutions and subsequently averaging their weights. Performance is evaluated on a multi-source dataset including holdout and external components. RESULTS: We find that model soups improve generalisation and calibration on the external component while maintaining performance on the holdout component. For robustness, we observe performance improvements for pertubated test images, while the performance on corrupted test images remains on par. CONCLUSIONS: Overall, souping for skin cancer classifiers has a positive effect on generalisation, robustness and calibration. It is easy for practitioners to implement and by combining multiple models into a single model, complexity is reduced. This could be an important factor in achieving clinical applicability, as less complexity generally means more transparency.
@article{Maron2022-ld,
abstract = {BACKGROUND: Image-based cancer classifiers suffer from a variety of problems which negatively affect their performance. For example, variation in image brightness or different cameras can already suffice to diminish performance. Ensemble solutions, where multiple model predictions are combined into one, can improve these problems. However, ensembles are computationally intensive and less transparent to practitioners than single model solutions. Constructing model soups, by averaging the weights of multiple models into a single model, could circumvent these limitations while still improving performance. OBJECTIVE: To investigate the performance of model soups for a dermoscopic melanoma-nevus skin cancer classification task with respect to (1) generalisation to images from other clinics, (2) robustness against small image changes and (3) calibration such that the confidences correspond closely to the actual predictive uncertainties. METHODS: We construct model soups by fine-tuning pre-trained models on seven different image resolutions and subsequently averaging their weights. Performance is evaluated on a multi-source dataset including holdout and external components. RESULTS: We find that model soups improve generalisation and calibration on the external component while maintaining performance on the holdout component. For robustness, we observe performance improvements for pertubated test images, while the performance on corrupted test images remains on par. CONCLUSIONS: Overall, souping for skin cancer classifiers has a positive effect on generalisation, robustness and calibration. It is easy for practitioners to implement and by combining multiple models into a single model, complexity is reduced. This could be an important factor in achieving clinical applicability, as less complexity generally means more transparency.},
added-at = {2024-09-10T11:54:51.000+0200},
author = {Maron, Roman C and Hekler, Achim and Haggenm{\"u}ller, Sarah and von Kalle, Christof and Utikal, Jochen S and M{\"u}ller, Verena and Gaiser, Maria and Meier, Friedegund and Hobelsberger, Sarah and Gellrich, Frank F and Sergon, Mildred and Hauschild, Axel and French, Lars E and Heinzerling, Lucie and Schlager, Justin G and Ghoreschi, Kamran and Schlaak, Max and Hilke, Franz J and Poch, Gabriela and Korsing, S{\"o}ren and Berking, Carola and Heppt, Markus V and Erdmann, Michael and Haferkamp, Sebastian and Schadendorf, Dirk and Sondermann, Wiebke and Goebeler, Matthias and Schilling, Bastian and Kather, Jakob N and Fr{\"o}hling, Stefan and Lipka, Daniel B and Krieghoff-Henning, Eva and Brinker, Titus J},
biburl = {https://puma.scadsai.uni-leipzig.de/bibtex/239f18326265e54d3e9d2d666fa35a590/scadsfct},
copyright = {http://creativecommons.org/licenses/by/4.0/},
interhash = {3b4670e8f433f7f2d091bf364fdaed3a},
intrahash = {39f18326265e54d3e9d2d666fa35a590},
journal = {Eur. J. Cancer},
keywords = {Artificial Calibration; Deep Dermatology; Ensembles; Generalisation; Melanoma; Model Nevus; Robustness intelligence; learning; soups; topic_lifescience},
language = {en},
month = sep,
pages = {307--316},
publisher = {Elsevier BV},
timestamp = {2024-09-10T12:00:15.000+0200},
title = {Model soups improve performance of dermoscopic skin cancer classifiers},
volume = 173,
year = 2022
}
