Coignion, T, Quinton, C., & Rouvoy, R. (2025). When Faster Isn’t Greener: The Hidden Costs of LLM-Based Code Optimization. ASE 2025 (Core Rank A*) (to Be Published).
@inproceedings{coignionFasterIsntGreener2025,
title = {When Faster Isn’t Greener: The Hidden Costs of LLM-Based Code Optimization},
booktitle = {ASE 2025 (Core Rank A*) (to be published)},
author = {Coignion, Tristan and Quinton, Cl{\'e}ment and Rouvoy, Romain},
year = {2025},
file = {../papers/crinkles-ase2025.pdf},
replication = {https://doi.org/10.5281/zenodo.15526180}
}
Coignion, T, Quinton, C., & Rouvoy, R. (2024). A Performance Study of LLM-Generated Code on Leetcode. Proceedings of the 28th International Conference on Evaluation and Assessment in Software Engineering, 79–89.
@inproceedings{coignionPerformanceStudyLLMGenerated2024,
title = {A {{Performance Study}} of {{LLM-Generated Code}} on {{Leetcode}}},
booktitle = {Proceedings of the 28th {{International Conference}} on {{Evaluation}} and {{Assessment}} in {{Software Engineering}}},
author = {Coignion, Tristan and Quinton, Cl{\'e}ment and Rouvoy, Romain},
year = {2024},
month = jun,
pages = {79--89},
publisher = {ACM},
address = {Salerno Italy},
doi = {10.1145/3661167.3661221},
urldate = {2024-10-04},
isbn = {979-8-4007-1701-7},
langid = {english},
file = {https://arxiv.org/pdf/2407.21579},
replication = {https://doi.org/10.5281/zenodo.7898304}
}
Preprints
Zine, N., Coignion, T, Stoico, V., Quinton, C., Malavolta, I., Rouvoy, R., & Lago, P. (2026). Attention to Detail: Evaluating Energy, Performance, and Accuracy Trade-offs Across vLLM Configurations (Issue arXiv:2607.09172). arXiv.
@unpublished{coignionAttentionDetailvLLM2026,
title = {Attention to {{Detail}}: {{Evaluating}} Energy, Performance, and {{Accuracy}} Trade-offs {{Across}} v{{LLM Configurations}}},
shorttitle = {Attention to {{Detail}}},
author = {Zine, Nada and Coignion, Tristan and Stoico, Vincenzo and Quinton, Cl{\'e}ment and Malavolta, Ivano and Rouvoy, Romain and Lago, Patricia},
year = {2026},
month = jul,
number = {arXiv:2607.09172},
eprint = {2607.09172},
primaryclass = {cs},
publisher = {arXiv},
doi = {2607.09172},
urldate = {2026-07-17},
archiveprefix = {arXiv},
keywords = {Computer Science - Software Engineering,Computer Science - Artificial Intelligence,Computer Science - Performance},
file = {https://arxiv.org/pdf/2607.09172},
replication = {https://doi.org/10.5281/zenodo.19253906}
}
Large Language Models are reshaping how software is developed and maintained. They are typically deployed in production using inference engines such as vLLM, which can efficiently serve pre-trained, highly configurable models. While prior work has focused on model architectures and hardware acceleration, the impact of inference engine configuration on energy consumption, performance, and output quality remains poorly understood. In this paper, we present a large-scale controlled study of three selected vLLM configuration options: attention kernel type, prefix caching, and chunked prefill. We evaluate all combinations of these configurations across 5 open-weight LLMs and 5 diverse inference tasks, totaling 9,000 runs and 93,600 measures. We analyze energy consumption, latency, and accuracy, and examine both main effects and interaction effects between configuration options and tasks. Our results show that the studied configuration options significantly impact energy and performance, mainly driven by attention type and prefix caching, while chunked prefill has a limited effect under the default vLLM serving configuration and evaluated workloads. These effects are highly model- and workload-dependent, and no configuration is universally optimal. We further show that model choice dominates global trade-offs, while configuration tuning provides local improvements along the Pareto frontier. Unexpectedly, inference options can also affect model accuracy.
Coignion, T, Quinton, C., & Rouvoy, R. (2024). Green My LLM: Studying the Key Factors Affecting the Energy Consumption of Code Assistants (Issue arXiv:2411.11892). arXiv.
@unpublished{coignionGreenMyLLM2024,
title = {Green {{My LLM}}: {{Studying}} the Key Factors Affecting the Energy Consumption of Code Assistants},
shorttitle = {Green {{My LLM}}},
author = {Coignion, Tristan and Quinton, Cl{\'e}ment and Rouvoy, Romain},
year = {2024},
month = nov,
number = {arXiv:2411.11892},
eprint = {2411.11892},
primaryclass = {cs},
publisher = {arXiv},
doi = {2411.11892},
urldate = {2025-03-03},
archiveprefix = {arXiv},
keywords = {Computer Science - Artificial Intelligence,Computer Science - Software Engineering},
file = {/papers/green_my_llm_paper.pdf},
replication = {https://doi.org/10.5281/zenodo.11503860}
}
In recent years,Large Language Models (LLMs) have significantly improved in generating high-quality code, enabling their integration into developers’ Integrated Development Environments (IDEs) as code assistants. These assistants, such as GitHub Copilot, deliver real-time code suggestions and can greatly enhance developers’ productivity. However, the environmental impact of these tools, in particular their energy consumption, remains a key concern. This paper investigates the energy consumption of LLM-based code assistants by simulating developer interactions with GitHub Copilot and analyzing various configuration factors. We collected a dataset of development traces from 20 developers and conducted extensive software project development simulations to measure energy usage under different scenarios. Our findings reveal that the energy consumption and performance of code assistants are influenced by various factors, such as the number of concurrent developers, model size, quantization methods, and the use of streaming. Notably, a substantial portion of generation requests made by GitHub Copilot is either canceled or rejected by developers, indicating a potential area for reducing wasted computations. Based on these findings, we share actionable insights into optimizing configurations for different use cases, demonstrating that careful adjustments can lead to significant energy savings.