Knowledge Distillation in Image Generation Models: Leveraging Powerful Generative Models to Enhance Smaller Models

@article{1708190,
      author = {Shivam Singh},
      title = {Knowledge Distillation in Image Generation Models: Leveraging Powerful Generative Models to Enhance Smaller Models},
      journal = {Iconic Research And Engineering Journals},
      year = {2025},
      volume = {8},
      number = {11},
      pages = {31-42},
      issn = {2456-8880},
      url = {https://www.irejournals.com/formatedpaper/1708190.pdf},
      abstract = {This paper presents an innovative framework for improving computationally constrained image generation models by distilling knowledge from more powerful but resource-intensive models. We demonstrate that Stable Diffusion XL (SDXL) can generate high-fidelity dog images that effectively train a smaller Stable Diffusion 1.5 model via Low-Rank Adaptation (LoRA). Our method eliminates the need for real-world data collection while achieving significant improvements in perceptual quality (31.46% SSIM increase in standard poses, p < 0.001) and structural accuracy. Through extensive evaluation using multiple metrics (SSIM, MSE, histogram similarity, perceptual hash similarity, FID score, and LPIPS), we reveal that knowledge transfer between diffusion models follows a hierarchical pattern where coarse structural features transfer more readily than fine details. We observe context- dependent performance variations, with dramatic improvement in standard poses and challenging scenarios but limitations in closeup details. Our findings demonstrate that extremely parameter- efficient adaptation (2.8MB) can achieve substantial quality improvements in resource-constrained environments, offering a promising pathway toward self-improving AI ecosystems with bidirectional knowledge flow between models of different capabilities.},
      month = {May},
  }