( = Paper PDF,
= Presentation slides,
= Presentation video)
Lizhi Liao; Simon Eismann; Heng Li; Cor-Paul Bezemer; Diego Costa; André van Hoorn; Weiyi Shang
Early Detection of Performance Regressions by Bridging Local Performance Data and Architectural Models Inproceedings
International Conference on Software Engineering (ICSE), 2025.
BibTeX | Tags: Performance, Performance analysis, Performance engineering, Performance evaluation, Performance regressions, Performance testing
@inproceedings{Liao_ICSE2025,
title = {Early Detection of Performance Regressions by Bridging Local Performance Data and Architectural Models},
author = {Lizhi Liao and Simon Eismann and Heng Li and Cor-Paul Bezemer and Diego Costa and André van Hoorn and Weiyi Shang},
year = {2025},
date = {2025-08-15},
urldate = {2025-08-15},
booktitle = {International Conference on Software Engineering (ICSE)},
keywords = {Performance, Performance analysis, Performance engineering, Performance evaluation, Performance regressions, Performance testing},
pubstate = {published},
tppubtype = {inproceedings}
}
Mikael Sabuhi
Strategies For Building Performant Containerized Applications PhD Thesis
2023.
Abstract | BibTeX | Tags: Docker, Docker Hub, Microservices, Performance, Performance analysis, Performance engineering
@phdthesis{phd_mikael,
title = {Strategies For Building Performant Containerized Applications},
author = {Mikael Sabuhi},
year = {2023},
date = {2023-09-25},
urldate = {2023-09-25},
abstract = {The evolution of cloud computing in the last decade has offered unprecedented access to sizable, configurable computing resources with minimal management effort. Containerization of applications, particularly through Docker, has been pivotal in this progression. As modern software increasingly relies on various cloud services, designing performant cloud applications has emerged as a critical concern. Key attributes of such applications include reliability, scalability, efficiency, fault tolerance, and responsiveness. This thesis seeks to address the challenges intrinsic to creating performant cloud applications by developing strategies aimed at achieving these characteristics through: 1) the application of autoscaling techniques to enhance scalability, efficiency, and responsiveness; 2) the introduction of a methodology for assessing the impact of Docker image upgrades on containerized applications to prevent performance degradation; and 3) the utilization of microservices architecture to develop scalable, reliable, and fault-tolerant cloud applications. In our initial research, we propose a pioneering approach to optimize the performance and resource usage of containerized cloud applications using adaptive controllers grounded in control theory. Our methodology harnesses the capacity of neural networks to capture the intrinsic non-linearity of these applications, and adapts the parameters of a proportional-integral-derivative (PID) controller to accommodate environmental changes. The outcomes demonstrate significant enhancements in resource utilization and a reduction in service level agreement violations, surpassing the performance of other examined autoscaling techniques. In the subsequent study, we present a method to evaluate the performance implications of Docker image upgrades on cloud software systems and their correlation with application dependencies. Our case study of 90 official WordPress images underscores the need for comprehensive performance testing before upgrades, the importance of maintaining a performance repository for reporting test results, and the potential benefits of extending semantic versioning to encompass performance modifications. This investigation encourages an enlightened approach to Docker image management, promoting enhanced cloud application performance. Lastly, we introduce Micro-FL, a fault-tolerant federated learning framework crafted to enhance the reliability and scalability of cloud-based machine learning platforms. By incorporating a microservices-based architecture within Docker containers, Micro-FL overcomes challenges typically associated with federated learning, such as resource constraints, scalability, and system faults. Performance assessments demonstrate Micro-FL’s capability to efficiently manage faults and streamline federated learning processes, offering a more robust and scalable solution for federated learning. The research work presented in this thesis provides deep insights, actionable recommendations, and effective and thoroughly evaluated approaches for building performant cloud applications.
},
keywords = {Docker, Docker Hub, Microservices, Performance, Performance analysis, Performance engineering},
pubstate = {published},
tppubtype = {phdthesis}
}
Simon Eismann; Diego Costa; Lizhi Liao; Cor-Paul Bezemer; Weiyi Shang; André van Hoorn; Samuel Kounev
A Case Study on the Stability of Performance Tests for Serverless Applications Journal Article
Journal of Systems and Software, 2022.
Abstract | BibTeX | Tags: Performance engineering, Performance regressions, Performance testing, Serverless
@article{EismannJSS2022,
title = {A Case Study on the Stability of Performance Tests for Serverless Applications},
author = {Simon Eismann and Diego Costa and Lizhi Liao and Cor-Paul Bezemer and Weiyi Shang and André van Hoorn and Samuel Kounev},
year = {2022},
date = {2022-03-17},
urldate = {2022-03-17},
journal = {Journal of Systems and Software},
abstract = {Context. While in serverless computing, application resource management and operational concerns are generally delegated to the cloud provider, ensuring that serverless applications meet their performance requirements is still a responsibility of the developers. Performance testing is a commonly used performance assessment practice; however, it traditionally requires visibility of the resource environment.
Objective. In this study, we investigate whether performance tests of serverless applications are stable, that is, if their results are reproducible, and what implications the serverless paradigm has for performance tests.
Method. We conduct a case study where we collect two datasets of performance test results: (a) repetitions of performance tests for varying memory size and load intensities and (b) three repetitions of the same performance test every day for ten months.
Results. We find that performance tests of serverless applications are comparatively stable if conducted on the same day. However, we also observe short-term performance variations and frequent long-term performance changes.
Conclusion. Performance tests for serverless applications can be stable; however, the serverless model impacts the planning, execution, and analysis of performance tests.},
keywords = {Performance engineering, Performance regressions, Performance testing, Serverless},
pubstate = {published},
tppubtype = {article}
}
Objective. In this study, we investigate whether performance tests of serverless applications are stable, that is, if their results are reproducible, and what implications the serverless paradigm has for performance tests.
Method. We conduct a case study where we collect two datasets of performance test results: (a) repetitions of performance tests for varying memory size and load intensities and (b) three repetitions of the same performance test every day for ten months.
Results. We find that performance tests of serverless applications are comparatively stable if conducted on the same day. However, we also observe short-term performance variations and frequent long-term performance changes.
Conclusion. Performance tests for serverless applications can be stable; however, the serverless model impacts the planning, execution, and analysis of performance tests.
Philipp Leitner; Cor-Paul Bezemer
An Exploratory Study of the State of Practice of Performance Testing in Java-based Open Source Projects Inproceedings
The International Conference on Performance Engineering (ICPE), pp. 373–384, ACM/SPEC, 2017.
Abstract | BibTeX | Tags: Empirical software engineering, Mining software repositories, Open source, Performance engineering, Performance testing
@inproceedings{leitner16oss,
title = {An Exploratory Study of the State of Practice of Performance Testing in Java-based Open Source Projects},
author = {Philipp Leitner and Cor-Paul Bezemer},
year = {2017},
date = {2017-04-22},
urldate = {2017-04-22},
booktitle = {The International Conference on Performance Engineering (ICPE)},
pages = {373--384},
publisher = {ACM/SPEC},
abstract = {The usage of open source (OS) software is nowadays widespread across many industries and domains. While the functional quality of OS projects is considered to be up to par with that of closed-source software, much is unknown about the quality in terms of non-functional attributes, such as
performance. One challenge for OS developers is that, unlike for functional testing, there is a lack of accepted best practices for performance testing.
To reveal the state of practice of performance testing in OS projects, we conduct an exploratory study on 111 Java-based OS projects from GitHub. We study the performance tests of these projects from five perspectives: (1) the developers, (2) size, (3) organization and (4) types of performance tests
and (5) the tooling used for performance testing.
First, in a quantitative study we show that writing performance tests is not a popular task in OS projects: performance tests form only a small portion of the test suite, are rarely updated, and are usually maintained by a small group of core project developers. Second, we show through a qualitative study that even though many projects are aware that they need performance tests, developers appear to struggle implementing them. We argue that future performance testing frameworks should provider better support for low-friction testing, for instance via non-parameterized methods
or performance test generation, as well as focus on a tight integration with standard continuous integration tooling.},
keywords = {Empirical software engineering, Mining software repositories, Open source, Performance engineering, Performance testing},
pubstate = {published},
tppubtype = {inproceedings}
}
performance. One challenge for OS developers is that, unlike for functional testing, there is a lack of accepted best practices for performance testing.
To reveal the state of practice of performance testing in OS projects, we conduct an exploratory study on 111 Java-based OS projects from GitHub. We study the performance tests of these projects from five perspectives: (1) the developers, (2) size, (3) organization and (4) types of performance tests
and (5) the tooling used for performance testing.
First, in a quantitative study we show that writing performance tests is not a popular task in OS projects: performance tests form only a small portion of the test suite, are rarely updated, and are usually maintained by a small group of core project developers. Second, we show through a qualitative study that even though many projects are aware that they need performance tests, developers appear to struggle implementing them. We argue that future performance testing frameworks should provider better support for low-friction testing, for instance via non-parameterized methods
or performance test generation, as well as focus on a tight integration with standard continuous integration tooling.