@phdthesis{phd_taesiri,

title = {Leveraging Foundation Models for Video Game Quality Assurance},

author = {Mohammad Reza Taesiri },

year  = {2024},

date = {2024-09-25},

abstract = {The video game industry has become a powerhouse in the global entertainment econ-

omy. Creating engaging, high-quality games demands intricate development processes

and significant resources. As projects grow in complexity and scale, developers often

grapple with demanding schedules, tight deadlines, and the risk of burnout. These

pressures highlight the need for more efficient development strategies, with quality

assurance (QA) emerging as a critical area for optimization.

Artificial Intelligence (AI) has the potential to address these challenges by en-

hancing the game QA processes in large gaming companies. Specifically, foundation

models—large pre-trained AI models—offer promising applications to improve these

processes. Exploring novel uses of these advanced AI models could reveal their poten-

tial and limitations in optimizing game development workflows, potentially alleviating

some of the industry’s pressing issues and facilitating the creation of high-quality, en-

gaging games.

In this thesis, my goal is to improve video game testing processes by leveraging

foundation models to ensure the final product reaches a desirable quality. I explore

new opportunities that foundation models bring to game testing, from searching for

instances of game bugs within video repositories to assisting human testers in catching

bugs, through three studies:

First, I investigate the utility of image-text foundation models in retrieving game-

play videos. In this study, I create a video search engine designed to help developers

efficiently search video repositories for examples of video game bugs using textual

descriptions. For example, developers can find all instances of a bug by using a tex-

tual description of the bug, such as a horse flying in the air. This study lays the

groundwork for AI-based game QA processes, with results demonstrating significant

potential.

Next, I introduce GlitchBench, a benchmarking dataset of video game glitches

and anomalies designed to assess state-of-the-art large multimodal models, such as

GPT-4V, in detecting and understanding game bugs. This extensive dataset includes

a wide range of images depicting various glitches, sourced from both online platforms

and synthetic sets created within the Unity game engine. GlitchBench includes both

common and rare glitches encountered in the video game quality assurance process.

The findings from this study highlight both the promise and limitations of existing

models, particularly in unusual and rare cases.

Lastly, I introduce VideoGameBunny, a large multimodal model specifically

trained for video game content, accompanied by a dataset of 389,565 image-instruction

pairs. My analysis demonstrates that VideoGameBunny outperforms much larger

models in video game understanding tasks while using 4.2× fewer parameters. This

result underscores the effectiveness and promise of using a high-quality dataset to

improve models’ understanding of video games, thus making them more effective in

the game QA process.

Future work should focus on enhancing the generalization and robustness of AI

models in the gaming context, particularly through better integration of vision and

language components. This integration could be achieved using either early or late fu-

sion methods. For late fusion methods, where two pre-trained models are connected,

better alignment between these components can be achieved through improved train-

ing data and strategies. Alternatively, early fusion techniques, which involve training

both components simultaneously to enhance their integration, can overcome many

issues that existing models have.},

keywords = {Computer games, Computer vision, Game development, Game testing, Gameplay videos, Machine learning, Software quality},

pubstate = {published},

tppubtype = {phdthesis}

}

@phdthesis{phd_haoli,

title = {Investigating the Quality of Bindings for Machine Learning Libraries in Software Package Ecosystems},

author = {Hao Li },

year  = {2024},

date = {2024-08-21},

urldate = {2024-08-21},

abstract = {Machine learning (ML) has revolutionized many domains, with developers often re-

lying on open source ML libraries to integrate ML capabilities into their projects.

However, these libraries primarily support a single programming language, limiting

their availability for projects in other languages. Bindings serve as bridges between

programming languages by providing interfaces to ML libraries. This thesis investi-

gates the quality of bindings for ML libraries in software package ecosystems, focusing

on their maintenance and software quality.

The first study presented in this thesis introduces BindFind, an automated ap-

proach to identify bindings and link them with their corresponding host libraries

across various software package ecosystems. By analyzing 2,436 bindings for 546 ML

libraries, we find that most bindings are community-maintained, with npm being the

most popular choice for publishing these bindings. The analysis reveals that these

bindings usually cover a limited range of releases from their host library and experi-

ence significant delays in supporting new releases.

In the second study, we investigate the usage and rationale behind release-level

deprecation in bindings for ML libraries within the Cargo and npm ecosystems. We

discover that bindings in Cargo have a higher percentage of deprecated releases com-

pared to general packages, while the percentages of deprecated releases and general

packages are similar in npm. The primary reasons for deprecation are package re-

moval or replacement and defects in both ecosystems. We also identify the issue of

implicitly deprecated releases in Cargo due to deprecation propagation through the

dependency network.

The third study evaluates the impact of using different bindings on the software

quality of ML systems through experiments on model training and inference using

TensorFlow and PyTorch across four programming languages. The results show that

models trained with one binding perform consistently in inference tasks when utilized

with another binding. Furthermore, non-default bindings can outperform the default

Python bindings in specific tasks without sacrificing accuracy. We also find significant

differences in inference times across bindings, highlighting the benefits of choosing ap-

propriate bindings based on specific performance requirements to maximize efficiency

in ML projects.

The work presented in this thesis provides deep insights, actionable recommenda-

tions, and effective and thoroughly evaluated approaches for assessing and improving

the quality of bindings for ML libraries in software package ecosystems.},

keywords = {Machine learning, Software Ecosystem, Software quality},

pubstate = {published},

tppubtype = {phdthesis}

}