TY  - JOUR
AU  - Manam, Karthik Babu
PY  - 2026
DA  - 2026/07/14
TI  - Scalable Software Engineering Architecture for AI-Enabled ETL Pipelines Using Event-Driven Microservices
JO  - ICCK Journal of Software Engineering
T2  - ICCK Journal of Software Engineering
JF  - ICCK Journal of Software Engineering
VL  - 2
IS  - 3
SP  - 169
EP  - 184
DO  - 10.62762/JSE.2026.382038
UR  - https://www.icck.org/article/abs/JSE.2026.382038
KW  - Event-Driven architecture
KW  - microservices engineering
KW  - scalable ETL pipelines
KW  - AI-Enabled data transformation
KW  - distributed data processing
KW  - predictive load balancing
KW  - anomaly detection
KW  - container orchestration
KW  - software engineering frameworks
KW  - data quality management
AB  - As data ecosystems become more diverse and time-critical, traditional monolithic ETL pipelines face challenges to meet the demands of modern data engineering workloads in terms of scalability, adaptability, and operational resilience. In this paper, we introduce an event-driven microservices approach to orchestrate and deploy AI-based ETL (ETL = Extraction, Transformation, and Loading) pipelines in a Kubernetes-managed environment that includes the following components: asynchronous orchestration using Apache Kafka, hybrid anomaly detection, adaptive schema inference, and predictive load balancing. The proposed architecture breaks the ETL processing into loosely coupled services, which can be deployed and scaled independently, and incorporates data quality intelligence into the transformation layer. Two explicit baselines are used for experimental evaluation: (1) a traditional monolithic batch-processing ETL pipeline with sequential execution of the stages performed without horizontal scaling, and (2) an event-driven microservices pipeline, which uses Apache Kafka orchestration but no AI-enabled optimization modules. The framework is able to achieve 39.7% improvement in throughput (4,820 vs. 3,450 records/sec) and 96.3% anomaly detection accuracy, while reducing average E2E latency by 14.6% (245 vs. 287 ms). The framework's throughput is 288.7% higher than the baseline of ETL monolithic, and the average latency drops 72.5% to 245 ms compared to 892 ms for the baseline. An architectural assessment also shows that the system is more modular, loosely coupled, more fault isolated, and more flexible to deploy; all of which are important software quality attributes. The results indicate the proposed architecture as a potentially reusable reference to create scalable and intelligent ETL systems in enterprise data processing environments and emphasize the necessity of generalization in multi-domain validation.
SN  - 3069-1834
PB  - Institute of Central Computation and Knowledge
LA  - English
ER  - 

@article{Manam2026Scalable,
  author = {Karthik Babu Manam},
  title = {Scalable Software Engineering Architecture for AI-Enabled ETL Pipelines Using Event-Driven Microservices},
  journal = {ICCK Journal of Software Engineering},
  year = {2026},
  volume = {2},
  number = {3},
  pages = {169-184},
  doi = {10.62762/JSE.2026.382038},
  url = {https://www.icck.org/article/abs/JSE.2026.382038},
  abstract = {As data ecosystems become more diverse and time-critical, traditional monolithic ETL pipelines face challenges to meet the demands of modern data engineering workloads in terms of scalability, adaptability, and operational resilience. In this paper, we introduce an event-driven microservices approach to orchestrate and deploy AI-based ETL (ETL = Extraction, Transformation, and Loading) pipelines in a Kubernetes-managed environment that includes the following components: asynchronous orchestration using Apache Kafka, hybrid anomaly detection, adaptive schema inference, and predictive load balancing. The proposed architecture breaks the ETL processing into loosely coupled services, which can be deployed and scaled independently, and incorporates data quality intelligence into the transformation layer. Two explicit baselines are used for experimental evaluation: (1) a traditional monolithic batch-processing ETL pipeline with sequential execution of the stages performed without horizontal scaling, and (2) an event-driven microservices pipeline, which uses Apache Kafka orchestration but no AI-enabled optimization modules. The framework is able to achieve 39.7\% improvement in throughput (4,820 vs. 3,450 records/sec) and 96.3\% anomaly detection accuracy, while reducing average E2E latency by 14.6\% (245 vs. 287 ms). The framework's throughput is 288.7\% higher than the baseline of ETL monolithic, and the average latency drops 72.5\% to 245 ms compared to 892 ms for the baseline. An architectural assessment also shows that the system is more modular, loosely coupled, more fault isolated, and more flexible to deploy; all of which are important software quality attributes. The results indicate the proposed architecture as a potentially reusable reference to create scalable and intelligent ETL systems in enterprise data processing environments and emphasize the necessity of generalization in multi-domain validation.},
  keywords = {Event-Driven architecture, microservices engineering, scalable ETL pipelines, AI-Enabled data transformation, distributed data processing, predictive load balancing, anomaly detection, container orchestration, software engineering frameworks, data quality management},
  issn = {3069-1834},
  publisher = {Institute of Central Computation and Knowledge}
}