Conditional generative models such as DALL-E and Stable Diffusion generate images based on a user-defined text, the prompt. Finding and refining prompts that produce a desired image has become the art of prompt engineering. Generative models do not provide a built-in retrieval model for a user's information need expressed through prompts. In light of an extensive literature review, we reframe prompt engineering for generative models as interactive text-based retrieval on a novel kind of ``infinite index''. We apply these insights for the first time in a case study on image generation for game design with an expert. Finally, we envision how active learning may help to guide the retrieval of generated images.
%0 Journal Article
%1 Deckers2022-pr
%A Deckers, Niklas
%A Fröbe, Maik
%A Kiesel, Johannes
%A Pandolfo, Gianluca
%A Schröder, Christopher
%A Stein, Benno
%A Potthast, Martin
%D 2022
%I arXiv
%K
%T The infinite index: Information retrieval on generative text-to-image models
%X Conditional generative models such as DALL-E and Stable Diffusion generate images based on a user-defined text, the prompt. Finding and refining prompts that produce a desired image has become the art of prompt engineering. Generative models do not provide a built-in retrieval model for a user's information need expressed through prompts. In light of an extensive literature review, we reframe prompt engineering for generative models as interactive text-based retrieval on a novel kind of ``infinite index''. We apply these insights for the first time in a case study on image generation for game design with an expert. Finally, we envision how active learning may help to guide the retrieval of generated images.
@article{Deckers2022-pr,
abstract = {Conditional generative models such as DALL-E and Stable Diffusion generate images based on a user-defined text, the prompt. Finding and refining prompts that produce a desired image has become the art of prompt engineering. Generative models do not provide a built-in retrieval model for a user's information need expressed through prompts. In light of an extensive literature review, we reframe prompt engineering for generative models as interactive text-based retrieval on a novel kind of ``infinite index''. We apply these insights for the first time in a case study on image generation for game design with an expert. Finally, we envision how active learning may help to guide the retrieval of generated images.},
added-at = {2024-09-10T10:41:24.000+0200},
author = {Deckers, Niklas and Fr{\"o}be, Maik and Kiesel, Johannes and Pandolfo, Gianluca and Schr{\"o}der, Christopher and Stein, Benno and Potthast, Martin},
biburl = {https://puma.scadsai.uni-leipzig.de/bibtex/2456a7c36606519aad166f49b39cc6581/scadsfct},
interhash = {c037ab5acc187f278c23f845035652fa},
intrahash = {456a7c36606519aad166f49b39cc6581},
keywords = {},
publisher = {arXiv},
timestamp = {2024-09-10T10:47:32.000+0200},
title = {The infinite index: Information retrieval on generative text-to-image models},
year = 2022
}
