Bidirectional Vision-Language Generation: Synthesizing Segmentation Masks and Captions with a Unified Cross-Modal Transformer

@article{1722526,
      author = {Camille Dubois, Arjun Nair, Sofia Petrova},
      title = {Bidirectional Vision-Language Generation: Synthesizing Segmentation Masks and Captions with a Unified Cross-Modal Transformer},
      journal = {Iconic Research And Engineering Journals},
      year = {2023},
      volume = {7},
      number = {5},
      pages = {475-480},
      issn = {2456-8880},
      url = {https://www.irejournals.com/formatedpaper/1722526.pdf},
      abstract = {We present a unified cross-modal transformer that generates in both directions between images and language: producing semantic segmentation masks from captions and captions from masks within a single model. A shared cross-modal attention stack, built on a Swin-B vision encoder and a BERT text encoder, aligns the two modalities in a common space. On Cityscapes and PASCAL-Context the model reaches 69.7% mIoU for text-to-mask and a BLEU-4 of 33 for mask-to-text, outperforming strong unidirectional baselines.},
      month = {November},
  }