TY  - JOUR
AU  - Qi, Wuqiang
AU  - Zhang, Zhuoqun
AU  - Wang, Zhishe
PY  - 2024
DA  - 2024/12/31
TI  - DMFuse: Diffusion Model Guided Cross-Attention Learning for Infrared and Visible Image Fusion
JO  - Chinese Journal of Information Fusion
T2  - Chinese Journal of Information Fusion
JF  - Chinese Journal of Information Fusion
VL  - 1
IS  - 3
SP  - 226
EP  - 242
DO  - 10.62762/CJIF.2024.655617
UR  - https://www.icck.org/article/abs/CJIF.2024.655617
KW  - image fusion
KW  - diffusion model
KW  - feature interaction
KW  - attention mechanism
KW  - deep generative model
AB  - Image fusion aims to integrate complementary information from different sensors into a single fused output for superior visual description and scene understanding. The existing GAN-based fusion methods generally suffer from multiple challenges, such as unexplainable mechanism, unstable training, and mode collapse, which may affect the fusion quality. To overcome these limitations, this paper introduces a diffusion model guided cross-attention learning network, termed as DMFuse, for infrared and visible image fusion. Firstly, to improve the diffusion inference efficiency, we compress the quadruple channels of the denoising UNet network to achieve more efficient and robust model for fusion tasks. After that, we employ the pre-trained diffusion model as an autoencoder and incorporate its strong generative priors to further train the following fusion network. This design allows the generated diffusion features to effectively showcase high-quality distribution mapping ability. In addition, we devise a cross-attention interactive fusion module to establish the long-range dependencies from local diffusion features. This module integrates the global interactions to improve the complementary characteristics of different modalities. Finally, we propose a multi-level decoder network to reconstruct the fused output. Extensive experiments on fusion tasks and downstream applications, including object detection and semantic segmentation, indicate that the proposed model yields promising performance while maintaining competitive computational efficiency. The code and data are available at https://github.com/Zhishe-Wang/DMFuse.
SN  - 2998-3371
PB  - Institute of Central Computation and Knowledge
LA  - English
ER  - 

@article{Qi2024DMFuse,
  author = {Wuqiang Qi and Zhuoqun Zhang and Zhishe Wang},
  title = {DMFuse: Diffusion Model Guided Cross-Attention Learning for Infrared and Visible Image Fusion},
  journal = {Chinese Journal of Information Fusion},
  year = {2024},
  volume = {1},
  number = {3},
  pages = {226-242},
  doi = {10.62762/CJIF.2024.655617},
  url = {https://www.icck.org/article/abs/CJIF.2024.655617},
  abstract = {Image fusion aims to integrate complementary information from different sensors into a single fused output for superior visual description and scene understanding. The existing GAN-based fusion methods generally suffer from multiple challenges, such as unexplainable mechanism, unstable training, and mode collapse, which may affect the fusion quality. To overcome these limitations, this paper introduces a diffusion model guided cross-attention learning network, termed as DMFuse, for infrared and visible image fusion. Firstly, to improve the diffusion inference efficiency, we compress the quadruple channels of the denoising UNet network to achieve more efficient and robust model for fusion tasks. After that, we employ the pre-trained diffusion model as an autoencoder and incorporate its strong generative priors to further train the following fusion network. This design allows the generated diffusion features to effectively showcase high-quality distribution mapping ability. In addition, we devise a cross-attention interactive fusion module to establish the long-range dependencies from local diffusion features. This module integrates the global interactions to improve the complementary characteristics of different modalities. Finally, we propose a multi-level decoder network to reconstruct the fused output. Extensive experiments on fusion tasks and downstream applications, including object detection and semantic segmentation, indicate that the proposed model yields promising performance while maintaining competitive computational efficiency. The code and data are available at https://github.com/Zhishe-Wang/DMFuse.},
  keywords = {image fusion, diffusion model, feature interaction, attention mechanism, deep generative model},
  issn = {2998-3371},
  publisher = {Institute of Central Computation and Knowledge}
}