%0 Conference Paper %1 Goens2019-yn %A Goens, Andrés %A Brauckmann, Alexander %A Ertel, Sebastian %A Cummins, Chris %A Leather, Hugh %A Castrillon, Jeronimo %B Proceedings of the 3rd ACM SIGPLAN International Workshop on Machine Learning and Programming Languages %C New York, NY, USA %D 2019 %I ACM %K %T A case study on machine learning for synthesizing benchmarks %X Good benchmarks are hard to find because they require a substantial effort to keep them representative for the constantly changing challenges of a particular field. Synthetic benchmarks are a common approach to deal with this, and methods from machine learning are natural candidates for synthetic benchmark generation. In this paper we investigate the usefulness of machine learning in the prominent CLgen benchmark generator. We re-evaluate CLgen by comparing the benchmarks generated by the model with the raw data used to train it. This re-evaluation indicates that, for the use case considered, machine learning did not yield additional benefit over a simpler method using the raw data. We investigate the reasons for this and provide further insights into the challenges the problem could pose for potential future generators.

@inproceedings{Goens2019-yn, abstract = {Good benchmarks are hard to find because they require a substantial effort to keep them representative for the constantly changing challenges of a particular field. Synthetic benchmarks are a common approach to deal with this, and methods from machine learning are natural candidates for synthetic benchmark generation. In this paper we investigate the usefulness of machine learning in the prominent CLgen benchmark generator. We re-evaluate CLgen by comparing the benchmarks generated by the model with the raw data used to train it. This re-evaluation indicates that, for the use case considered, machine learning did not yield additional benefit over a simpler method using the raw data. We investigate the reasons for this and provide further insights into the challenges the problem could pose for potential future generators.}, added-at = {2024-09-10T11:56:37.000+0200}, address = {New York, NY, USA}, author = {Goens, Andr{\'e}s and Brauckmann, Alexander and Ertel, Sebastian and Cummins, Chris and Leather, Hugh and Castrillon, Jeronimo}, biburl = {https://puma.scadsai.uni-leipzig.de/bibtex/2a4332e0ae15fd174e6b524de0564a64b/scadsfct}, booktitle = {Proceedings of the 3rd {ACM} {SIGPLAN} International Workshop on Machine Learning and Programming Languages}, conference = {PLDI '19: 40th ACM SIGPLAN Conference on Programming Language Design and Implementation}, copyright = {http://www.acm.org/publications/policies/copyright\_policy\#Background}, interhash = {aaebef82ab7c6766bf476cc8e05f443a}, intrahash = {a4332e0ae15fd174e6b524de0564a64b}, keywords = {}, location = {Phoenix AZ USA}, month = jun, publisher = {ACM}, timestamp = {2024-09-10T15:15:57.000+0200}, title = {A case study on machine learning for synthesizing benchmarks}, year = 2019 }